2018-11-24 16:55:19,850 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 16:56:23,819 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 16:57:02,063 - synapse.http.server - 112 - ERROR - GET-465- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-24 16:57:17,085 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 16:57:51,318 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 16:57:55,156 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 16:58:15,585 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 16:59:00,600 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 16:59:39,174 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:00:26,495 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:02:41,053 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:03:23,321 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:03:43,459 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:04:04,744 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:05:47,193 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:06:31,925 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:07:52,317 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:09:12,190 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:10:43,129 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:14:29,432 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:15:30,873 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:16:10,616 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:16:16,223 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:16:37,620 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:17:52,173 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:18:58,395 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:19:49,061 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:21:56,167 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:22:08,150 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:22:13,743 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:22:14,782 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:22:47,545 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:23:31,150 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:25:24,473 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:26:18,839 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:26:22,475 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:26:58,007 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:28:19,072 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:28:22,143 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:29:22,517 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:30:15,468 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:31:28,760 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:31:36,190 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:31:54,100 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:33:03,181 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:33:05,908 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:34:43,519 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:34:55,176 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:35:43,230 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:35:46,010 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:36:05,899 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:05,902 - synapse.federation.transport.server - 359 - ERROR - PUT-6372- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:05,913 - synapse.http.server - 112 - ERROR - PUT-6372- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:06,055 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:06,059 - synapse.federation.transport.server - 359 - ERROR - PUT-6373- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:06,064 - synapse.http.server - 112 - ERROR - PUT-6373- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:08,018 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:08,020 - synapse.federation.transport.server - 359 - ERROR - PUT-6374- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:08,024 - synapse.http.server - 112 - ERROR - PUT-6374- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:09,634 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:09,637 - synapse.federation.transport.server - 359 - ERROR - PUT-6376- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:09,643 - synapse.http.server - 112 - ERROR - PUT-6376- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:10,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:10,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-595- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:10,878 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:10,881 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-303- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:11,836 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:11,840 - synapse.http.server - 112 - ERROR - PUT-6377- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:17,624 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 17:36:17,628 - synapse.federation.transport.server - 359 - ERROR - PUT-6378- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 17:36:17,634 - synapse.http.server - 112 - ERROR - PUT-6378- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 17:36:54,906 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:44:55,153 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:46:27,255 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:46:34,243 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:47:41,273 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:47:59,991 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:48:33,160 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:49:38,555 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:49:48,302 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:52:57,982 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:53:55,703 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:56:40,189 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 17:57:58,009 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 18:01:32,794 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 18:05:17,636 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:17,639 - synapse.federation.transport.server - 359 - ERROR - PUT-10083- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:17,643 - synapse.http.server - 112 - ERROR - PUT-10083- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:17,925 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:17,928 - synapse.federation.transport.server - 359 - ERROR - PUT-10084- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:17,931 - synapse.http.server - 112 - ERROR - PUT-10084- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:19,354 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:19,358 - synapse.federation.transport.server - 359 - ERROR - PUT-10085- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:19,360 - synapse.http.server - 112 - ERROR - PUT-10085- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:19,848 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:19,851 - synapse.federation.transport.server - 359 - ERROR - PUT-10086- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:19,854 - synapse.http.server - 112 - ERROR - PUT-10086- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-940- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,542 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,547 - synapse.federation.transport.server - 359 - ERROR - PUT-10087- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,555 - synapse.http.server - 112 - ERROR - PUT-10087- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,624 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,627 - synapse.metrics.background_process_metrics - 203 - ERROR - event_push_action_stream_orderings-7- Background process 'event_push_action_stream_orderings' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,776 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,779 - synapse.federation.transport.server - 359 - ERROR - PUT-10088- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:05:20,785 - synapse.http.server - 112 - ERROR - PUT-10088- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,483 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,519 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,534 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,538 - synapse.http.server - 112 - ERROR - GET-10219- Failed handle request via : : Traceback (most recent call last): --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 167, in on_GET full_state=full_state File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 239, in wait_for_sync_for_user sync_config, since_token, timeout, full_state, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 269, in _wait_for_sync_for_user from_token=since_token, File "/usr/local/lib/python3.5/dist-packages/synapse/notifier.py", line 350, in wait_for_events result = yield callback(prev_token, current_token) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 929, in generate_sync_result sync_result_builder, account_data_by_room File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1274, in _generate_sync_entry_for_rooms res = yield self._get_rooms_changed(sync_result_builder, ignored_users) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1493, in _get_rooms_changed limit=timeline_limit + 1, File "/usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py", line 212, in get_room_events_stream_for_rooms ], consumeErrors=True)) twisted.internet.defer.FirstError: FirstError[#0, [Failure instance: Traceback: : connection already closed /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:501:errback /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:568:_startRunCallbacks /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:654:_runCallbacks /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1475:gotResult --- --- /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1416:_inlineCallbacks /usr/local/lib/python3.5/dist-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator /usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py:279:get_room_events_stream_for_room /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1416:_inlineCallbacks /usr/local/lib/python3.5/dist-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator /usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py:325:runInteraction /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1416:_inlineCallbacks /usr/local/lib/python3.5/dist-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator /usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py:374:runWithConnection /usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py:250:inContext /usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py:266: /usr/local/lib/python3.5/dist-packages/twisted/python/context.py:122:callWithContext /usr/local/lib/python3.5/dist-packages/twisted/python/context.py:85:callWithContext /usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py:306:_runWithConnection /usr/local/lib/python3.5/dist-packages/twisted/python/compat.py:464:reraise /usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py:297:_runWithConnection /usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py:370:inner_func /usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py:241:_new_transaction ]] 2018-11-24 18:06:57,542 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 18:06:57,544 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/user_erasure_store.py", line 72, in are_users_erased "are_users_erased", _get_erased_users, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,552 - synapse.http.server - 112 - ERROR - GET-10220- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 501, in errback self._startRunCallbacks(fail) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks self._runCallbacks() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 167, in on_GET full_state=full_state File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 239, in wait_for_sync_for_user sync_config, since_token, timeout, full_state, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 269, in _wait_for_sync_for_user from_token=since_token, File "/usr/local/lib/python3.5/dist-packages/synapse/notifier.py", line 350, in wait_for_events result = yield callback(prev_token, current_token) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 929, in generate_sync_result sync_result_builder, account_data_by_room File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1298, in _generate_sync_entry_for_rooms yield concurrently_execute(handle_room_entries, room_entries, 10) File "/usr/local/lib/python3.5/dist-packages/synapse/util/async_helpers.py", line 149, in _concurrently_execute_inner yield func(next(it)) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1642, in _generate_room_entry newly_joined_room=newly_joined, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 390, in _load_filtered_recents always_include_ids=current_state_ids, File "/usr/local/lib/python3.5/dist-packages/synapse/visibility.py", line 89, in filter_events_for_client erased_senders = yield store.are_users_erased((e.sender for e in events)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/user_erasure_store.py", line 72, in are_users_erased "are_users_erased", _get_erased_users, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,563 - synapse.metrics.background_process_metrics - 203 - ERROR - user_directory.notify_new_event-1580- Background process 'user_directory.notify_new_event' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/user_directory.py", line 114, in process yield self._unsafe_process() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/user_directory.py", line 163, in _unsafe_process yield self.store.update_user_directory_stream_pos(self.pos) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,571 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,574 - synapse.push.pusherpool - 154 - ERROR - PUT-10222-$1543082812386681pKmdw:matrix.org- Exception in pusher on_new_notifications Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/push/pusherpool.py", line 145, in on_new_notifications min_stream_id, max_stream_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_push_actions.py", line 178, in get_push_action_users_in_range ret = yield self.runInteraction("get_push_action_users_in_range", f) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,588 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,595 - synapse.federation.federation_server - 227 - ERROR - PUT-10222-$1543082812386681pKmdw:matrix.org- Failed to handle PDU $1543082812386681pKmdw:matrix.org: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 216, in process_pdus_for_room origin, pdu File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 640, in _handle_received_pdu origin, pdu, sent_to_us_directly=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 421, in on_receive_pdu auth_chain=auth_chain, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 604, in _process_received_pdu room = yield self.store.get_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,603 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,606 - synapse.metrics.background_process_metrics - 203 - ERROR - process_event_queue_for_federation-60- Background process 'process_event_queue_for_federation' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transaction_queue.py", line 160, in _process_event_queue_loop last_token = yield self.store.get_federation_out_pos("events") File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,927 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:57,981 - synapse.http.server - 112 - ERROR - GET-10225- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 236, in _respond_remote_thumbnail media_info = yield self.media_repo.get_remote_media_info(server_name, media_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 282, in get_remote_media_info server_name, media_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 306, in _get_remote_media_impl server_name, media_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:06:58,087 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:06:58,090 - synapse.http.server - 112 - ERROR - GET-10228- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 90, in _respond_local_thumbnail media_info = yield self.store.get_local_media(media_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:07:02,721 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:07:06,660 - synapse.http.server - 112 - ERROR - GET-10233- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 90, in _respond_local_thumbnail media_info = yield self.store.get_local_media(media_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:13:25,820 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:25,883 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:25,896 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:25,925 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:25,938 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:25,945 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:26,024 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:26,046 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:13:26,056 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:37:02,955 - synapse.http.server - 112 - ERROR - GET-13793- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 236, in _respond_remote_thumbnail media_info = yield self.media_repo.get_remote_media_info(server_name, media_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 282, in get_remote_media_info server_name, media_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 332, in _get_remote_media_impl server_name, media_id, file_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 442, in _download_remote_file filesystem_id=file_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 246, in _new_transaction r = func(txn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 453, in _simple_insert_txn txn.execute(sql, vals) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 87, in execute self._do_execute(self.txn.execute, sql, *args) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 117, in _do_execute sql, *args psycopg2.IntegrityError: duplicate key value violates unique constraint "remote_media_cache_media_origin_media_id_key" DETAIL: Key (media_origin, media_id)=(matrix.org, VIEJRFddhKNXQLugggEyBElS) already exists. 2018-11-24 18:44:36,847 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:36,850 - synapse.federation.transport.server - 359 - ERROR - PUT-15436- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:36,854 - synapse.http.server - 112 - ERROR - PUT-15436- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,092 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,095 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-32835- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,617 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,620 - synapse.http.server - 112 - ERROR - PUT-15437- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,816 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,819 - synapse.federation.transport.server - 359 - ERROR - PUT-15438- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,824 - synapse.http.server - 112 - ERROR - PUT-15438- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,991 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:38,993 - synapse.federation.transport.server - 359 - ERROR - PUT-15439- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:39,002 - synapse.http.server - 112 - ERROR - PUT-15439- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:39,207 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:39,210 - synapse.federation.transport.server - 359 - ERROR - PUT-15440- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:39,217 - synapse.http.server - 112 - ERROR - PUT-15440- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:40,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:40,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1409- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:40,868 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:40,871 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-714- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:41,700 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:41,703 - synapse.federation.transport.server - 359 - ERROR - PUT-15441- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:41,706 - synapse.http.server - 112 - ERROR - PUT-15441- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:42,157 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:42,160 - synapse.federation.transport.server - 359 - ERROR - PUT-15442- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:44:42,163 - synapse.http.server - 112 - ERROR - PUT-15442- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:26,355 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:26,358 - synapse.federation.transport.server - 359 - ERROR - PUT-15739- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:26,361 - synapse.http.server - 112 - ERROR - PUT-15739- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:27,030 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:27,035 - synapse.http.server - 112 - ERROR - PUT-15740- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:28,503 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:28,505 - synapse.federation.transport.server - 359 - ERROR - PUT-15741- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:28,509 - synapse.http.server - 112 - ERROR - PUT-15741- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:30,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:30,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1443- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:31,102 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:31,105 - synapse.federation.transport.server - 359 - ERROR - PUT-15742- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:31,107 - synapse.http.server - 112 - ERROR - PUT-15742- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:35,688 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:35,691 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-32650- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:36,215 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:36,218 - synapse.federation.transport.server - 359 - ERROR - PUT-15743- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:47:36,221 - synapse.http.server - 112 - ERROR - PUT-15743- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:47:36,410 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:36,413 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-32479- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:47:40,551 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:47:40,554 - synapse.federation.transport.server - 359 - ERROR - PUT-15748- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:47:40,559 - synapse.http.server - 112 - ERROR - PUT-15748- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:47:40,944 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-732- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:47:43,293 - synapse.federation.transport.server - 359 - ERROR - PUT-15749- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:47:43,297 - synapse.http.server - 112 - ERROR - PUT-15749- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:00,880 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:00,883 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-734- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:03,012 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:03,016 - synapse.federation.transport.server - 359 - ERROR - PUT-15770- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:03,021 - synapse.http.server - 112 - ERROR - PUT-15770- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:06,221 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:06,224 - synapse.federation.transport.server - 359 - ERROR - PUT-15773- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:06,228 - synapse.http.server - 112 - ERROR - PUT-15773- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:06,568 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:06,572 - synapse.http.server - 112 - ERROR - PUT-15774- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:28,154 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:28,158 - synapse.federation.transport.server - 359 - ERROR - PUT-15789- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:28,164 - synapse.http.server - 112 - ERROR - PUT-15789- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:30,499 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:30,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1455- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:30,870 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:30,872 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-737- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:30,984 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:30,988 - synapse.http.server - 112 - ERROR - PUT-15790- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:32,343 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:32,346 - synapse.http.server - 112 - ERROR - PUT-15791- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:34,738 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:34,741 - synapse.http.server - 112 - ERROR - PUT-15792- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:35,155 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:35,158 - synapse.federation.transport.server - 359 - ERROR - PUT-15793- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:35,163 - synapse.http.server - 112 - ERROR - PUT-15793- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:37,621 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:37,624 - synapse.federation.transport.server - 359 - ERROR - PUT-15794- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:37,628 - synapse.http.server - 112 - ERROR - PUT-15794- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:38,611 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:38,615 - synapse.federation.federation_server - 809 - ERROR - PUT-15795- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:38,627 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:38,630 - synapse.federation.transport.server - 359 - ERROR - PUT-15795- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:38,636 - synapse.http.server - 112 - ERROR - PUT-15795- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:40,132 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 18:48:40,135 - synapse.federation.transport.server - 359 - ERROR - PUT-15797- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:40,139 - synapse.http.server - 112 - ERROR - PUT-15797- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:40,956 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-738- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:43,443 - synapse.federation.federation_server - 809 - ERROR - PUT-15799- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 603, in _persist_and_notify parties = yield get_interested_parties(self.store, states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 1297, in get_interested_parties plist = yield store.get_presence_list_observers_accepted(state.user_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 261, in get_presence_list_observers_accepted desc="get_presence_list_accepted", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:43,602 - synapse.federation.transport.server - 359 - ERROR - PUT-15799- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:43,607 - synapse.http.server - 112 - ERROR - PUT-15799- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:43,627 - synapse.federation.transport.server - 359 - ERROR - PUT-15800- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:43,633 - synapse.http.server - 112 - ERROR - PUT-15800- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:43,972 - synapse.federation.transport.server - 359 - ERROR - PUT-15803- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:43,977 - synapse.http.server - 112 - ERROR - PUT-15803- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:45,608 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1458- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:49,508 - synapse.federation.transport.server - 359 - ERROR - PUT-15807- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:49,510 - synapse.http.server - 112 - ERROR - PUT-15807- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:50,569 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1459- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:55,227 - synapse.federation.transport.server - 359 - ERROR - PUT-15810- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:48:55,230 - synapse.http.server - 112 - ERROR - PUT-15810- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 18:57:43,782 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 18:59:01,581 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:06:34,521 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:07:22,379 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:20:32,602 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:21:32,407 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:22:02,401 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:02,403 - synapse.federation.transport.server - 359 - ERROR - PUT-19581- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:02,406 - synapse.http.server - 112 - ERROR - PUT-19581- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:02,565 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:02,568 - synapse.federation.transport.server - 359 - ERROR - PUT-19582- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:02,571 - synapse.http.server - 112 - ERROR - PUT-19582- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:04,729 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:04,732 - synapse.federation.transport.server - 359 - ERROR - PUT-19583- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:04,735 - synapse.http.server - 112 - ERROR - PUT-19583- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,191 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,194 - synapse.federation.transport.server - 359 - ERROR - PUT-19584- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,199 - synapse.http.server - 112 - ERROR - PUT-19584- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,501 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1851- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,598 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,601 - synapse.federation.transport.server - 359 - ERROR - PUT-19585- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:05,605 - synapse.http.server - 112 - ERROR - PUT-19585- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:06,135 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:06,143 - synapse.federation.transport.server - 359 - ERROR - PUT-19586- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:06,150 - synapse.http.server - 112 - ERROR - PUT-19586- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:06,987 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:06,991 - synapse.federation.transport.server - 359 - ERROR - PUT-19587- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:06,995 - synapse.http.server - 112 - ERROR - PUT-19587- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:07,209 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:07,212 - synapse.federation.transport.server - 359 - ERROR - PUT-19589- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:07,218 - synapse.http.server - 112 - ERROR - PUT-19589- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:08,021 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:08,024 - synapse.federation.transport.server - 359 - ERROR - PUT-19590- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:08,029 - synapse.http.server - 112 - ERROR - PUT-19590- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:08,616 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:08,621 - synapse.http.server - 112 - ERROR - GET-19591- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/profile.py", line 37, in on_GET user, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 122, in get_displayname target_user.localpart File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:08,758 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:08,762 - synapse.http.server - 112 - ERROR - GET-19592- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/profile.py", line 80, in on_GET user, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 178, in get_avatar_url target_user.localpart File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:09,226 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:09,229 - synapse.federation.transport.server - 359 - ERROR - PUT-19595- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:09,233 - synapse.http.server - 112 - ERROR - PUT-19595- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:09,701 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:09,704 - synapse.federation.transport.server - 359 - ERROR - PUT-19596- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:09,709 - synapse.http.server - 112 - ERROR - PUT-19596- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:10,022 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:10,025 - synapse.federation.transport.server - 359 - ERROR - PUT-19597- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:10,033 - synapse.http.server - 112 - ERROR - PUT-19597- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:10,510 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:10,513 - synapse.federation.transport.server - 359 - ERROR - PUT-19598- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:10,519 - synapse.http.server - 112 - ERROR - PUT-19598- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:10,541 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:10,544 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1852- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:10,966 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:22:10,969 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-939- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:13,092 - synapse.federation.transport.server - 359 - ERROR - PUT-19601- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:13,094 - synapse.http.server - 112 - ERROR - PUT-19601- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:14,262 - synapse.federation.transport.server - 359 - ERROR - PUT-19602- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:14,266 - synapse.http.server - 112 - ERROR - PUT-19602- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:15,044 - synapse.federation.transport.server - 359 - ERROR - PUT-19603- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:15,048 - synapse.http.server - 112 - ERROR - PUT-19603- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:15,092 - synapse.federation.transport.server - 359 - ERROR - PUT-19604- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:15,095 - synapse.http.server - 112 - ERROR - PUT-19604- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:17,819 - synapse.federation.transport.server - 359 - ERROR - PUT-19609- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:17,829 - synapse.http.server - 112 - ERROR - PUT-19609- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:19,911 - synapse.federation.transport.server - 359 - ERROR - PUT-19611- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:19,916 - synapse.http.server - 112 - ERROR - PUT-19611- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:22:55,088 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:23:15,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:15,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1865- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:15,637 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:15,640 - synapse.federation.transport.server - 359 - ERROR - PUT-19788- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:15,644 - synapse.http.server - 112 - ERROR - PUT-19788- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:16,883 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:16,886 - synapse.federation.transport.server - 359 - ERROR - PUT-19789- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:16,889 - synapse.http.server - 112 - ERROR - PUT-19789- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:17,710 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:17,714 - synapse.http.server - 112 - ERROR - PUT-19790- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:18,450 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:18,453 - synapse.http.server - 112 - ERROR - PUT-19791- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1866- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,819 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,821 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-157- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,858 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,863 - synapse.util - 136 - ERROR - - Looping call died Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 266, in _persist_unpersisted_changes for user_id in unpersisted File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,866 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:23:20,867 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 266, in _persist_unpersisted_changes for user_id in unpersisted File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,874 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:20,879 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-946- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:21,290 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:21,298 - synapse.http.server - 112 - ERROR - PUT-19792- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:23,693 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:23,697 - synapse.http.server - 112 - ERROR - PUT-19793- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:24,162 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:24,165 - synapse.http.server - 112 - ERROR - PUT-19794- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:24,610 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:24,613 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:24,616 - synapse.http.server - 112 - ERROR - PUT-19795- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:24,627 - synapse.http.server - 112 - ERROR - PUT-19796- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:25,509 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:25,512 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1867- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:26,327 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:26,331 - synapse.federation.transport.server - 359 - ERROR - PUT-19797- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:26,335 - synapse.http.server - 112 - ERROR - PUT-19797- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:27,454 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:27,458 - synapse.http.server - 112 - ERROR - PUT-19798- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:27,771 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:27,774 - synapse.http.server - 112 - ERROR - PUT-19799- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:28,999 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:29,003 - synapse.http.server - 112 - ERROR - PUT-19800- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:30,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1868- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:30,898 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-947- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:33,181 - synapse.http.server - 112 - ERROR - PUT-19802- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:35,091 - synapse.http.server - 112 - ERROR - PUT-19803- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:35,510 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1869- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:36,790 - synapse.http.server - 112 - ERROR - PUT-19807- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:37,796 - synapse.http.server - 112 - ERROR - PUT-19808- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:37,953 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:23:37,956 - synapse.federation.transport.server - 359 - ERROR - PUT-19809- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:37,961 - synapse.http.server - 112 - ERROR - PUT-19809- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:38,862 - synapse.http.server - 112 - ERROR - PUT-19810- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:40,300 - synapse.federation.transport.server - 359 - ERROR - PUT-19811- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:40,305 - synapse.http.server - 112 - ERROR - PUT-19811- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:40,446 - synapse.federation.transport.server - 359 - ERROR - PUT-19812- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:40,450 - synapse.http.server - 112 - ERROR - PUT-19812- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:40,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1870- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:40,643 - synapse.http.server - 112 - ERROR - PUT-19813- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:40,897 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-948- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:41,008 - synapse.federation.transport.server - 359 - ERROR - PUT-19814- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:41,012 - synapse.http.server - 112 - ERROR - PUT-19814- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:43,136 - synapse.http.server - 112 - ERROR - PUT-19815- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:43,737 - synapse.http.server - 112 - ERROR - PUT-19816- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:43,799 - synapse.http.server - 112 - ERROR - PUT-19817- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:44,726 - synapse.http.server - 112 - ERROR - PUT-19818- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:45,184 - synapse.http.server - 112 - ERROR - PUT-19819- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:45,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1871- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:45,667 - synapse.http.server - 112 - ERROR - PUT-19820- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:45,867 - synapse.http.server - 112 - ERROR - PUT-19821- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:46,026 - synapse.http.server - 112 - ERROR - PUT-19822- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:46,416 - synapse.federation.transport.server - 359 - ERROR - PUT-19823- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:46,419 - synapse.http.server - 112 - ERROR - PUT-19823- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:46,596 - synapse.http.server - 112 - ERROR - PUT-19824- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:47,803 - synapse.http.server - 112 - ERROR - PUT-19825- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:48,984 - synapse.federation.transport.server - 359 - ERROR - PUT-19826- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:48,989 - synapse.http.server - 112 - ERROR - PUT-19826- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:49,803 - synapse.http.server - 112 - ERROR - PUT-19827- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:50,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1872- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:50,556 - synapse.http.server - 112 - ERROR - PUT-19828- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:50,893 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-949- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:50,950 - synapse.http.server - 112 - ERROR - PUT-19829- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:51,343 - synapse.http.server - 112 - ERROR - PUT-19831- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:51,995 - synapse.http.server - 112 - ERROR - PUT-19832- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:53,328 - synapse.federation.transport.server - 359 - ERROR - PUT-19833- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:53,332 - synapse.http.server - 112 - ERROR - PUT-19833- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:53,430 - synapse.federation.transport.server - 359 - ERROR - PUT-19834- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:53,438 - synapse.http.server - 112 - ERROR - PUT-19834- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:55,033 - synapse.http.server - 112 - ERROR - PUT-19836- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:55,333 - synapse.http.server - 112 - ERROR - PUT-19839- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:55,431 - synapse.federation.transport.server - 359 - ERROR - PUT-19841- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:55,435 - synapse.http.server - 112 - ERROR - PUT-19841- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:55,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1873- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:57,329 - synapse.http.server - 112 - ERROR - PUT-19844- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:57,687 - synapse.federation.transport.server - 359 - ERROR - PUT-19845- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:57,691 - synapse.http.server - 112 - ERROR - PUT-19845- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:23:58,088 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:24:00,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1874- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:00,892 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-950- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:01,493 - synapse.http.server - 112 - ERROR - PUT-19850- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:01,781 - synapse.metrics.background_process_metrics - 203 - ERROR - fetch_events-210- Background process 'fetch_events' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:05,129 - synapse.federation.transport.server - 359 - ERROR - PUT-19851- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:05,131 - synapse.http.server - 112 - ERROR - PUT-19851- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:05,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1875- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:06,639 - synapse.federation.transport.server - 359 - ERROR - PUT-19852- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:06,643 - synapse.http.server - 112 - ERROR - PUT-19852- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:07,795 - synapse.http.server - 112 - ERROR - PUT-19854- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:09,298 - synapse.federation.transport.server - 359 - ERROR - PUT-19858- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:09,301 - synapse.http.server - 112 - ERROR - PUT-19858- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:10,439 - synapse.http.server - 112 - ERROR - GET-19859- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/profile.py", line 37, in on_GET user, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 122, in get_displayname target_user.localpart File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:10,667 - synapse.http.server - 112 - ERROR - GET-19862- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/account.py", line 317, in on_GET requester.user.to_string() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/registration.py", line 471, in user_get_threepids 'user_get_threepids' File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:11,761 - synapse.federation.transport.server - 359 - ERROR - PUT-19864- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:11,765 - synapse.http.server - 112 - ERROR - PUT-19864- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:44,673 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:44,675 - synapse.federation.transport.server - 359 - ERROR - PUT-19893- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:44,678 - synapse.http.server - 112 - ERROR - PUT-19893- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:45,243 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:45,246 - synapse.http.server - 112 - ERROR - PUT-19894- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:45,349 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:45,353 - synapse.http.server - 112 - ERROR - PUT-19892- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:45,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:45,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1883- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:47,446 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:47,448 - synapse.federation.transport.server - 359 - ERROR - PUT-19895- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:47,451 - synapse.http.server - 112 - ERROR - PUT-19895- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:47,885 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:47,887 - synapse.federation.transport.server - 359 - ERROR - PUT-19896- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:47,895 - synapse.http.server - 112 - ERROR - PUT-19896- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:50,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:50,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1884- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:50,870 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:50,872 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-955- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:54,308 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:54,321 - synapse.http.server - 112 - ERROR - GET-19898- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 159, in on_GET yield self.presence_handler.set_state(user, {"presence": set_presence}, True) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 751, in set_state yield self._update_states([prev_state.copy_and_replace(**new_fields)]) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:55,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:55,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1885- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:55,641 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:55,644 - synapse.http.server - 112 - ERROR - PUT-19899- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:57,356 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:57,361 - synapse.http.server - 112 - ERROR - PUT-19901- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:24:59,708 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:24:59,711 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39605- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:00,504 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:00,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1886- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:00,883 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:00,885 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-956- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:03,773 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:03,776 - synapse.federation.transport.server - 359 - ERROR - PUT-19902- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:03,778 - synapse.http.server - 112 - ERROR - PUT-19902- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:05,702 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:05,705 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1887- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:06,494 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:06,497 - synapse.http.server - 112 - ERROR - PUT-19903- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:09,226 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:09,228 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39431- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:10,184 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:10,189 - synapse.http.server - 112 - ERROR - GET-19904- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 167, in on_GET full_state=full_state File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 239, in wait_for_sync_for_user sync_config, since_token, timeout, full_state, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 261, in _wait_for_sync_for_user sync_config, since_token, full_state=full_state, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 925, in generate_sync_result sync_result_builder File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1130, in _generate_sync_entry_for_account_data since_token.account_data_key, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:10,511 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1888- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:10,878 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-957- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:11,426 - synapse.federation.transport.server - 359 - ERROR - PUT-19907- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:11,430 - synapse.http.server - 112 - ERROR - PUT-19907- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:14,085 - synapse.http.server - 112 - ERROR - PUT-19908- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:14,886 - synapse.http.server - 112 - ERROR - PUT-19909- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:15,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1889- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:17,270 - synapse.http.server - 112 - ERROR - PUT-19910- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:20,510 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1890- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:20,640 - synapse.metrics.background_process_metrics - 203 - ERROR - event_push_action_stream_orderings-15- Background process 'event_push_action_stream_orderings' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:20,864 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-159- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:20,902 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-958- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:20,956 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-159- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:21,101 - synapse.metrics.background_process_metrics - 203 - ERROR - generate_user_daily_visits-31- Background process 'generate_user_daily_visits' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:21,624 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-40990- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:23,727 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39912- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:25,332 - synapse.federation.transport.server - 359 - ERROR - PUT-19913- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:25,337 - synapse.http.server - 112 - ERROR - PUT-19913- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:25,503 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39795- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:25,627 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1891- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:27,762 - synapse.http.server - 112 - ERROR - PUT-19914- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:30,057 - synapse.federation.transport.server - 359 - ERROR - PUT-19915- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:30,060 - synapse.http.server - 112 - ERROR - PUT-19915- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:30,514 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1892- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:31,095 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-959- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:33,076 - synapse.metrics.background_process_metrics - 203 - ERROR - fetch_events-212- Background process 'fetch_events' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:33,085 - synapse.http.server - 112 - ERROR - GET-19919- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 413, in on_GET state_filter=StateFilter.from_types([(EventTypes.Member, None)]), File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 125, in get_state_events room_id, end_token=at_token.room_key, limit=1, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py", line 358, in get_recent_events_for_room room_id, limit, end_token, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py", line 395, in get_recent_event_ids_for_room room_id, from_token=end_token, limit=limit, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,364 - synapse.handlers.typing - 245 - ERROR - PUT-19916- Error pushing typing notif to remotes Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 221, in _push_remote users = yield self.state.get_current_user_in_room(member.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,373 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:25:34,374 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,377 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:25:34,378 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,380 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:25:34,382 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,387 - synapse.http.server - 112 - ERROR - PUT-19849- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 501, in errback self._startRunCallbacks(fail) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks self._runCallbacks() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 564, in create_new_client_event yield self.auth.add_auth_events(builder, context) File "/usr/local/lib/python3.5/dist-packages/synapse/api/auth.py", line 553, in add_auth_events auth_ids File "/usr/local/lib/python3.5/dist-packages/synapse/storage/signatures.py", line 59, in add_event_hashes event_ids File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,436 - synapse.federation.transport.server - 359 - ERROR - PUT-19922- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,443 - synapse.http.server - 112 - ERROR - PUT-19922- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:34,598 - synapse.http.server - 112 - ERROR - PUT-19924- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:35,363 - synapse.http.server - 112 - ERROR - GET-19917- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 479, in on_GET event_filter=event_filter, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/pagination.py", line 222, in get_messages room_id, max_topo File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 825, in maybe_backfill room_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:37,961 - synapse.federation.transport.server - 359 - ERROR - PUT-19925- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:37,970 - synapse.http.server - 112 - ERROR - PUT-19925- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:39,114 - synapse.federation.transport.server - 359 - ERROR - PUT-19926- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:39,119 - synapse.http.server - 112 - ERROR - PUT-19926- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:39,837 - synapse.http.server - 112 - ERROR - PUT-19927- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:41,496 - synapse.http.server - 112 - ERROR - PUT-19928- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:45,512 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1895- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:46,504 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-40045- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:48,913 - synapse.http.server - 112 - ERROR - PUT-19934- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:55,819 - synapse.http.server - 112 - ERROR - PUT-19935- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:55,883 - synapse.handlers.typing - 245 - ERROR - - Error pushing typing notif to remotes Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 221, in _push_remote users = yield self.state.get_current_user_in_room(member.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:57,284 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,293 - synapse.federation.transport.server - 359 - ERROR - PUT-19930- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:57,296 - synapse.http.server - 112 - ERROR - PUT-19930- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:57,300 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,303 - synapse.http.server - 112 - ERROR - GET-19929- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 167, in on_GET full_state=full_state File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 239, in wait_for_sync_for_user sync_config, since_token, timeout, full_state, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 269, in _wait_for_sync_for_user from_token=since_token, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/notifier.py", line 350, in wait_for_events result = yield callback(prev_token, current_token) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 925, in generate_sync_result sync_result_builder File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1130, in _generate_sync_entry_for_account_data since_token.account_data_key, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:57,311 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,313 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-960- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:57,315 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-40058- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,319 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1896- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,322 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:25:57,322 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,323 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:25:57,324 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,325 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:25:57,325 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,328 - synapse.http.server - 112 - ERROR - PUT-19931- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 501, in errback self._startRunCallbacks(fail) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks self._runCallbacks() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 564, in create_new_client_event yield self.auth.add_auth_events(builder, context) File "/usr/local/lib/python3.5/dist-packages/synapse/api/auth.py", line 553, in add_auth_events auth_ids File "/usr/local/lib/python3.5/dist-packages/synapse/storage/signatures.py", line 59, in add_event_hashes event_ids File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,382 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:25:57,384 - synapse.http.server - 112 - ERROR - PUT-19938- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:25:58,066 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39672- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:00,516 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1897- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:00,869 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:26:00,872 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-961- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:26:02,480 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:26:02,483 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39896- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:03,485 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:26:03,490 - synapse.http.server - 112 - ERROR - PUT-19939- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:04,160 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39832- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:05,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1898- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:05,871 - synapse.handlers.presence - 277 - ERROR - - Error updating presence Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 274, in _update_states_and_catch_exception res = yield self._update_states(new_states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:05,905 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-40027- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:07,325 - synapse.http.server - 112 - ERROR - PUT-19940- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:10,512 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1899- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:10,875 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-962- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:13,877 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:26:13,883 - synapse.http.server - 112 - ERROR - PUT-19941- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:13,980 - synapse.http.server - 112 - ERROR - PUT-19942- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:14,815 - synapse.federation.transport.server - 359 - ERROR - PUT-19943- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:14,823 - synapse.http.server - 112 - ERROR - PUT-19943- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:15,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1900- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:16,058 - synapse.http.server - 112 - ERROR - PUT-19944- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:17,073 - synapse.http.server - 112 - ERROR - PUT-19945- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:17,117 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39776- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:20,377 - synapse.http.server - 112 - ERROR - PUT-19946- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:20,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1901- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:20,833 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-160- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:20,862 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-160- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:20,891 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-963- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:20,994 - synapse.http.server - 112 - ERROR - PUT-19947- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:24,211 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-40037- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:25,528 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1902- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:29,496 - synapse.http.server - 112 - ERROR - PUT-19950- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:30,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1903- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:30,891 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-964- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:35,510 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1904- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:36,646 - synapse.http.server - 112 - ERROR - PUT-19951- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:37,672 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39646- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:40,509 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1905- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:40,891 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-965- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:43,085 - synapse.http.server - 112 - ERROR - PUT-19952- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:43,329 - synapse.federation.transport.server - 359 - ERROR - PUT-19953- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:43,335 - synapse.http.server - 112 - ERROR - PUT-19953- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:43,743 - synapse.federation.transport.server - 359 - ERROR - PUT-19954- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:43,747 - synapse.http.server - 112 - ERROR - PUT-19954- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:44,728 - synapse.federation.transport.server - 359 - ERROR - PUT-19955- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:44,731 - synapse.http.server - 112 - ERROR - PUT-19955- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:45,036 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39386- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:45,425 - synapse.federation.transport.server - 359 - ERROR - PUT-19956- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:45,430 - synapse.http.server - 112 - ERROR - PUT-19956- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:45,507 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1906- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:46,450 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39861- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:46,540 - synapse.http.server - 112 - ERROR - PUT-19957- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:48,169 - synapse.http.server - 112 - ERROR - PUT-19958- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:48,647 - synapse.federation.transport.server - 359 - ERROR - PUT-19959- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:48,650 - synapse.http.server - 112 - ERROR - PUT-19959- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:49,393 - synapse.http.server - 112 - ERROR - PUT-19960- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:50,513 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1907- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:50,887 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-966- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:52,431 - synapse.http.server - 112 - ERROR - PUT-19961- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:52,500 - synapse.http.server - 112 - ERROR - PUT-19962- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:52,888 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39946- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:53,069 - synapse.http.server - 112 - ERROR - PUT-19963- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:54,088 - synapse.http.server - 112 - ERROR - PUT-19964- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:55,367 - synapse.http.server - 112 - ERROR - PUT-19965- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:55,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1908- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:57,714 - synapse.handlers.typing - 245 - ERROR - PUT-19969- Error pushing typing notif to remotes Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 221, in _push_remote users = yield self.state.get_current_user_in_room(member.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:57,717 - synapse.handlers.typing - 245 - ERROR - PUT-19971- Error pushing typing notif to remotes Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 221, in _push_remote users = yield self.state.get_current_user_in_room(member.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:57,720 - synapse.handlers.typing - 245 - ERROR - PUT-19970- Error pushing typing notif to remotes Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 221, in _push_remote users = yield self.state.get_current_user_in_room(member.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:57,726 - synapse.http.server - 112 - ERROR - PUT-19968- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 766, in on_PUT yield self.presence_handler.bump_presence_active_time(requester.user) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 414, in bump_presence_active_time yield self._update_states([prev_state.copy_and_replace(**new_fields)]) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:26:59,415 - synapse.http.server - 112 - ERROR - PUT-19977- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:00,311 - synapse.handlers.typing - 245 - ERROR - PUT-19980- Error pushing typing notif to remotes Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 221, in _push_remote users = yield self.state.get_current_user_in_room(member.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:00,328 - synapse.http.server - 112 - ERROR - PUT-19981- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:00,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1909- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:00,882 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-967- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:01,253 - synapse.federation.transport.server - 359 - ERROR - PUT-19984- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:01,260 - synapse.http.server - 112 - ERROR - PUT-19984- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:02,900 - synapse.federation.transport.server - 359 - ERROR - PUT-19985- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:02,903 - synapse.http.server - 112 - ERROR - PUT-19985- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:04,413 - synapse.http.server - 112 - ERROR - PUT-19986- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:04,532 - synapse.crypto.keyring - 688 - ERROR - PUT-19987- Got Exception when downloading keys for jones.ninja: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:04,634 - synapse.federation.transport.server - 359 - ERROR - PUT-19988- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:04,638 - synapse.http.server - 112 - ERROR - PUT-19988- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:04,717 - synapse.http.server - 112 - ERROR - PUT-19989- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:04,836 - synapse.http.server - 112 - ERROR - PUT-19990- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:04,914 - synapse.http.server - 112 - ERROR - PUT-19991- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,011 - synapse.federation.transport.server - 359 - ERROR - PUT-19992- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,018 - synapse.http.server - 112 - ERROR - PUT-19992- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,047 - synapse.http.server - 112 - ERROR - PUT-19993- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,094 - synapse.http.server - 112 - ERROR - PUT-19994- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,190 - synapse.http.server - 112 - ERROR - PUT-19995- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,520 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1910- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,617 - synapse.http.server - 112 - ERROR - PUT-19996- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,669 - synapse.federation.transport.server - 359 - ERROR - PUT-19997- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,674 - synapse.http.server - 112 - ERROR - PUT-19997- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,682 - synapse.crypto.keyring - 688 - ERROR - PUT-19998- Got Exception when downloading keys for tadhack.vector.im: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,818 - synapse.http.server - 112 - ERROR - PUT-19999- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,824 - synapse.http.server - 112 - ERROR - PUT-20000- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:05,897 - synapse.http.server - 112 - ERROR - PUT-20001- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,207 - synapse.federation.transport.server - 359 - ERROR - PUT-20002- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,210 - synapse.http.server - 112 - ERROR - PUT-20002- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,259 - synapse.http.server - 112 - ERROR - PUT-20004- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,268 - synapse.http.server - 112 - ERROR - PUT-20003- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,404 - synapse.http.server - 112 - ERROR - PUT-20005- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,496 - synapse.http.server - 112 - ERROR - PUT-20006- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,595 - synapse.http.server - 112 - ERROR - PUT-20007- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,645 - synapse.http.server - 112 - ERROR - PUT-20008- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,698 - synapse.http.server - 112 - ERROR - PUT-20009- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,721 - synapse.crypto.keyring - 688 - ERROR - PUT-20010- Got Exception when downloading keys for matrix.meuschkescloud.de: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,739 - synapse.crypto.keyring - 688 - ERROR - PUT-20011- Got Exception when downloading keys for olelo.im: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,870 - synapse.http.server - 112 - ERROR - PUT-20013- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:06,898 - synapse.http.server - 112 - ERROR - PUT-20014- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,016 - synapse.http.server - 112 - ERROR - PUT-20015- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,052 - synapse.federation.transport.server - 359 - ERROR - PUT-20016- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,060 - synapse.http.server - 112 - ERROR - PUT-20016- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,170 - synapse.http.server - 112 - ERROR - PUT-20017- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,308 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-39639- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,442 - synapse.http.server - 112 - ERROR - PUT-20018- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,624 - synapse.http.server - 112 - ERROR - PUT-20019- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,685 - synapse.federation.transport.server - 359 - ERROR - PUT-20020- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,689 - synapse.http.server - 112 - ERROR - PUT-20020- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,741 - synapse.http.server - 112 - ERROR - PUT-20021- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,824 - synapse.http.server - 112 - ERROR - PUT-20022- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:07,974 - synapse.http.server - 112 - ERROR - PUT-20023- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,116 - synapse.http.server - 112 - ERROR - PUT-20024- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,237 - synapse.federation.transport.server - 359 - ERROR - PUT-20025- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,242 - synapse.http.server - 112 - ERROR - PUT-20025- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,479 - synapse.federation.transport.server - 359 - ERROR - PUT-20026- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,485 - synapse.http.server - 112 - ERROR - PUT-20026- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,571 - synapse.http.server - 112 - ERROR - PUT-20027- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,613 - synapse.http.server - 112 - ERROR - PUT-20028- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,682 - synapse.http.server - 112 - ERROR - PUT-20029- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,928 - synapse.http.server - 112 - ERROR - PUT-20030- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,958 - synapse.crypto.keyring - 688 - ERROR - PUT-20031- Got Exception when downloading keys for m.wfr.moe: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:08,999 - synapse.http.server - 112 - ERROR - PUT-20032- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:09,130 - synapse.http.server - 112 - ERROR - PUT-20033- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:09,505 - synapse.http.server - 112 - ERROR - PUT-20034- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:09,633 - synapse.http.server - 112 - ERROR - PUT-20036- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,198 - synapse.http.server - 112 - ERROR - PUT-20038- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,223 - synapse.http.server - 112 - ERROR - PUT-20039- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,228 - synapse.federation.transport.server - 359 - ERROR - PUT-20035- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,232 - synapse.http.server - 112 - ERROR - PUT-20035- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,237 - synapse.federation.transport.server - 359 - ERROR - PUT-20037- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,242 - synapse.http.server - 112 - ERROR - PUT-20037- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,295 - synapse.http.server - 112 - ERROR - PUT-20040- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,427 - synapse.http.server - 112 - ERROR - PUT-20044- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,433 - synapse.federation.transport.server - 359 - ERROR - PUT-20043- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,436 - synapse.http.server - 112 - ERROR - PUT-20043- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,470 - synapse.http.server - 112 - ERROR - PUT-20045- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:27:10,510 - synapse.http.server - 112 - ERROR - PUT-20046- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:11,006 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:11,138 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:11,711 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:11,976 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:11,985 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,308 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,409 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,457 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,487 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-1933- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,491 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-980- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,496 - synapse.federation.transport.server - 359 - ERROR - PUT-20293- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,500 - synapse.http.server - 112 - ERROR - PUT-20293- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,510 - synapse.federation.transport.server - 359 - ERROR - PUT-20295- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,520 - synapse.http.server - 112 - ERROR - PUT-20295- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,525 - synapse.federation.transport.server - 359 - ERROR - PUT-20296- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:12,527 - synapse.http.server - 112 - ERROR - PUT-20296- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:15,496 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:15,500 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:15,841 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:29:16,049 - synapse.federation.transport.server - 359 - ERROR - PUT-20298- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,054 - synapse.http.server - 112 - ERROR - PUT-20298- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,059 - synapse.federation.transport.server - 359 - ERROR - PUT-20300- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,064 - synapse.http.server - 112 - ERROR - PUT-20300- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,069 - synapse.federation.transport.server - 359 - ERROR - PUT-20301- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,074 - synapse.http.server - 112 - ERROR - PUT-20301- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,832 - synapse.federation.transport.server - 359 - ERROR - PUT-20302- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,836 - synapse.http.server - 112 - ERROR - PUT-20302- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,953 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-24 19:29:16,955 - twisted - 244 - CRITICAL - - Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 169, in get_presence_for_users desc="get_presence_for_users", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:16,957 - synapse.federation.federation_server - 809 - ERROR - PUT-20307- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:17,381 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-45258- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:29:17,550 - synapse.metrics.background_process_metrics - 203 - ERROR - user_directory.notify_new_event-2911- Background process 'user_directory.notify_new_event' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/user_directory.py", line 114, in process yield self._unsafe_process() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/user_directory.py", line 160, in _unsafe_process yield self._handle_deltas(deltas) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/user_directory.py", line 364, in _handle_deltas yield self._handle_new_user(room_id, state_key, profile) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/user_directory.py", line 453, in _handle_new_user row = yield self.store.get_user_in_directory(user_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:30:55,876 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:33:02,631 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:37:14,128 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:38:30,854 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:38:53,447 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,449 - synapse.federation.transport.server - 359 - ERROR - PUT-21244- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,453 - synapse.http.server - 112 - ERROR - PUT-21244- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,716 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,719 - synapse.federation.transport.server - 359 - ERROR - PUT-21245- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,726 - synapse.http.server - 112 - ERROR - PUT-21245- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,902 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,905 - synapse.federation.transport.server - 359 - ERROR - PUT-21246- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:53,909 - synapse.http.server - 112 - ERROR - PUT-21246- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:54,364 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:54,367 - synapse.federation.transport.server - 359 - ERROR - PUT-21247- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:54,372 - synapse.http.server - 112 - ERROR - PUT-21247- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:54,854 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:54,857 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-42771- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:55,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:55,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2047- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:55,926 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:55,929 - synapse.federation.transport.server - 359 - ERROR - PUT-21248- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:55,933 - synapse.http.server - 112 - ERROR - PUT-21248- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:56,275 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:56,279 - synapse.federation.transport.server - 359 - ERROR - PUT-21249- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:56,283 - synapse.http.server - 112 - ERROR - PUT-21249- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:58,344 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:58,347 - synapse.federation.transport.server - 359 - ERROR - PUT-21251- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:58,351 - synapse.http.server - 112 - ERROR - PUT-21251- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:58,982 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:58,986 - synapse.federation.transport.server - 359 - ERROR - PUT-21252- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:38:58,989 - synapse.http.server - 112 - ERROR - PUT-21252- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:07,621 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:07,624 - synapse.federation.transport.server - 359 - ERROR - PUT-21586- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:07,628 - synapse.http.server - 112 - ERROR - PUT-21586- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,059 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,062 - synapse.federation.transport.server - 359 - ERROR - PUT-21587- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,068 - synapse.http.server - 112 - ERROR - PUT-21587- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,250 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,257 - synapse.federation.transport.server - 359 - ERROR - PUT-21588- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,259 - synapse.http.server - 112 - ERROR - PUT-21588- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,575 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,580 - synapse.federation.transport.server - 359 - ERROR - PUT-21589- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,584 - synapse.http.server - 112 - ERROR - PUT-21589- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,779 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,782 - synapse.federation.transport.server - 359 - ERROR - PUT-21591- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:08,786 - synapse.http.server - 112 - ERROR - PUT-21591- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:11,362 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:11,367 - synapse.federation.transport.server - 359 - ERROR - PUT-21593- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:11,371 - synapse.http.server - 112 - ERROR - PUT-21593- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:12,144 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:12,147 - synapse.federation.transport.server - 359 - ERROR - PUT-21594- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:12,151 - synapse.http.server - 112 - ERROR - PUT-21594- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:12,225 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:41:12,228 - synapse.federation.transport.server - 359 - ERROR - PUT-21596- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:12,233 - synapse.http.server - 112 - ERROR - PUT-21596- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:12,955 - synapse.federation.transport.server - 359 - ERROR - PUT-21601- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:12,960 - synapse.http.server - 112 - ERROR - PUT-21601- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:13,279 - synapse.federation.federation_server - 809 - ERROR - PUT-21602- Failed to handle edu 'm.typing' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 264, in _recv_edu users = yield self.state.get_current_user_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:15,914 - synapse.federation.federation_server - 809 - ERROR - PUT-21607- Failed to handle edu 'm.typing' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/typing.py", line 264, in _recv_edu users = yield self.state.get_current_user_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 168, in get_current_user_in_room joined_users = yield self.store.get_joined_users_from_state(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:41:20,827 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-175- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:25,349 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:25,351 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-49035- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:25,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:25,499 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2089- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:26,823 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:26,828 - synapse.federation.transport.server - 359 - ERROR - PUT-21711- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:26,836 - synapse.http.server - 112 - ERROR - PUT-21711- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:28,241 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:28,244 - synapse.federation.transport.server - 359 - ERROR - PUT-21712- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:28,247 - synapse.http.server - 112 - ERROR - PUT-21712- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:29,926 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:29,929 - synapse.federation.transport.server - 359 - ERROR - PUT-21713- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:29,934 - synapse.http.server - 112 - ERROR - PUT-21713- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:30,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:30,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2090- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:30,792 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:30,800 - synapse.federation.transport.server - 359 - ERROR - PUT-21714- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:30,806 - synapse.http.server - 112 - ERROR - PUT-21714- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:30,868 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:30,872 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1060- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:34,457 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:34,460 - synapse.federation.transport.server - 359 - ERROR - PUT-21715- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:34,462 - synapse.http.server - 112 - ERROR - PUT-21715- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:34,630 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:34,632 - synapse.federation.transport.server - 359 - ERROR - PUT-21716- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:34,635 - synapse.http.server - 112 - ERROR - PUT-21716- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:35,638 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:35,639 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2091- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:38,072 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:38,077 - synapse.federation.transport.server - 359 - ERROR - PUT-21719- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:38,080 - synapse.http.server - 112 - ERROR - PUT-21719- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:38,354 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:38,357 - synapse.federation.transport.server - 359 - ERROR - PUT-21720- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:38,361 - synapse.http.server - 112 - ERROR - PUT-21720- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:47,434 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:47,437 - synapse.federation.transport.server - 359 - ERROR - PUT-21727- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:47,440 - synapse.http.server - 112 - ERROR - PUT-21727- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:48,029 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:48,032 - synapse.federation.transport.server - 359 - ERROR - PUT-21728- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:48,037 - synapse.http.server - 112 - ERROR - PUT-21728- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:48,773 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:48,776 - synapse.federation.transport.server - 359 - ERROR - PUT-21729- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:48,779 - synapse.http.server - 112 - ERROR - PUT-21729- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:50,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:50,499 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2094- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:50,870 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:50,873 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1062- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:51,265 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:51,268 - synapse.http.server - 112 - ERROR - PUT-21731- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:51,720 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:51,724 - synapse.federation.transport.server - 359 - ERROR - PUT-21732- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:51,733 - synapse.http.server - 112 - ERROR - PUT-21732- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:51,825 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:42:51,829 - synapse.federation.transport.server - 359 - ERROR - PUT-21733- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:51,833 - synapse.http.server - 112 - ERROR - PUT-21733- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:54,332 - synapse.federation.transport.server - 359 - ERROR - PUT-21738- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:42:54,335 - synapse.http.server - 112 - ERROR - PUT-21738- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:05,586 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:05,589 - synapse.federation.transport.server - 359 - ERROR - PUT-21751- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:05,595 - synapse.http.server - 112 - ERROR - PUT-21751- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:05,654 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:05,657 - synapse.federation.transport.server - 359 - ERROR - PUT-21753- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:05,662 - synapse.http.server - 112 - ERROR - PUT-21753- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:07,599 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:07,602 - synapse.federation.transport.server - 359 - ERROR - PUT-21754- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:07,608 - synapse.http.server - 112 - ERROR - PUT-21754- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:09,824 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:09,826 - synapse.federation.transport.server - 359 - ERROR - PUT-21757- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:09,829 - synapse.http.server - 112 - ERROR - PUT-21757- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:10,816 - synapse.federation.transport.server - 359 - ERROR - PUT-21760- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:10,820 - synapse.http.server - 112 - ERROR - PUT-21760- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:20,013 - synapse.federation.transport.server - 359 - ERROR - PUT-21764- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:20,019 - synapse.http.server - 112 - ERROR - PUT-21764- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:20,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:20,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2100- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:21,190 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:21,193 - synapse.http.server - 112 - ERROR - PUT-21766- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:22,413 - synapse.federation.transport.server - 359 - ERROR - PUT-21767- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:22,417 - synapse.http.server - 112 - ERROR - PUT-21767- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:27,775 - synapse.federation.transport.server - 359 - ERROR - PUT-21771- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:27,780 - synapse.http.server - 112 - ERROR - PUT-21771- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 19:43:47,437 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 19:43:47,441 - synapse.federation.federation_server - 809 - ERROR - PUT-21793- Failed to handle edu 'm.device_list_update' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/device.py", line 485, in incoming_device_list_update yield self._handle_device_updates(user_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/device.py", line 549, in _handle_device_updates user_id, [device_id for device_id, _, _, _ in pending_updates] File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/device.py", line 272, in notify_device_update user_id, device_ids, list(hosts) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/devices.py", line 678, in add_device_change_to_streams user_id, device_ids, hosts, stream_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 19:45:17,933 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:53:34,184 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:53:53,428 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:55:05,327 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:56:09,680 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:58:24,853 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 19:59:41,572 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:00:56,655 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,678 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,727 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,785 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,853 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,858 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,860 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,861 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,867 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:56,874 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:57,002 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,433 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,500 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,518 - synapse.federation.transport.server - 359 - ERROR - PUT-23543- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,522 - synapse.http.server - 112 - ERROR - PUT-23543- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,528 - synapse.federation.transport.server - 359 - ERROR - PUT-23545- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,535 - synapse.http.server - 112 - ERROR - PUT-23545- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,537 - synapse.federation.transport.server - 359 - ERROR - PUT-23546- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,542 - synapse.http.server - 112 - ERROR - PUT-23546- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,548 - synapse.federation.transport.server - 359 - ERROR - PUT-23544- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,553 - synapse.http.server - 112 - ERROR - PUT-23544- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,556 - synapse.federation.transport.server - 359 - ERROR - PUT-23547- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,562 - synapse.http.server - 112 - ERROR - PUT-23547- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,568 - synapse.federation.transport.server - 359 - ERROR - PUT-23548- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,573 - synapse.http.server - 112 - ERROR - PUT-23548- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:00:58,581 - synapse.federation.transport.server - 359 - ERROR - PUT-23549- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:00:58,585 - synapse.http.server - 112 - ERROR - PUT-23549- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:01:02,855 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:01:56,603 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:56,606 - synapse.federation.transport.server - 359 - ERROR - PUT-23709- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:56,609 - synapse.http.server - 112 - ERROR - PUT-23709- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:56,651 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:56,656 - synapse.federation.transport.server - 359 - ERROR - PUT-23708- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:56,660 - synapse.http.server - 112 - ERROR - PUT-23708- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:57,161 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:57,164 - synapse.federation.transport.server - 359 - ERROR - PUT-23710- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:57,170 - synapse.http.server - 112 - ERROR - PUT-23710- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:57,376 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:57,378 - synapse.federation.transport.server - 359 - ERROR - PUT-23711- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:57,384 - synapse.http.server - 112 - ERROR - PUT-23711- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:58,573 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:58,575 - synapse.federation.transport.server - 359 - ERROR - PUT-23712- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:58,579 - synapse.http.server - 112 - ERROR - PUT-23712- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:58,632 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:58,635 - synapse.federation.transport.server - 359 - ERROR - PUT-23713- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:58,640 - synapse.http.server - 112 - ERROR - PUT-23713- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,195 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,199 - synapse.federation.transport.server - 359 - ERROR - PUT-23714- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,202 - synapse.http.server - 112 - ERROR - PUT-23714- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,685 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,687 - synapse.federation.transport.server - 359 - ERROR - PUT-23715- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,691 - synapse.http.server - 112 - ERROR - PUT-23715- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,975 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,977 - synapse.federation.transport.server - 359 - ERROR - PUT-23716- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:01:59,982 - synapse.http.server - 112 - ERROR - PUT-23716- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:00,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:00,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2321- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:00,878 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:00,880 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1177- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:01,205 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:01,207 - synapse.federation.transport.server - 359 - ERROR - PUT-23717- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:01,209 - synapse.http.server - 112 - ERROR - PUT-23717- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:03,606 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:03,610 - synapse.federation.transport.server - 359 - ERROR - PUT-23718- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:03,612 - synapse.http.server - 112 - ERROR - PUT-23718- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:03,845 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:03,849 - synapse.federation.transport.server - 359 - ERROR - PUT-23719- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:03,851 - synapse.http.server - 112 - ERROR - PUT-23719- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:05,932 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:05,934 - synapse.federation.transport.server - 359 - ERROR - PUT-23720- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:05,937 - synapse.http.server - 112 - ERROR - PUT-23720- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:08,673 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:08,676 - synapse.federation.federation_server - 809 - ERROR - PUT-23721- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:08,685 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:08,688 - synapse.federation.transport.server - 359 - ERROR - PUT-23721- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:08,693 - synapse.http.server - 112 - ERROR - PUT-23721- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:10,275 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:10,281 - synapse.http.server - 112 - ERROR - POST-23722- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/pusher.py", line 112, in on_POST not_user_id=user.to_string() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/push/pusherpool.py", line 110, in remove_pushers_by_app_id_and_pushkey_not_user app_id, pushkey File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/pusher.py", line 97, in get_pushers_by ], desc="get_pushers_by" File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:10,352 - synapse.http.server - 112 - ERROR - GET-23727- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 159, in on_GET yield self.presence_handler.set_state(user, {"presence": set_presence}, True) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 751, in set_state yield self._update_states([prev_state.copy_and_replace(**new_fields)]) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:10,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2323- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:10,910 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1178- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:11,813 - synapse.federation.transport.server - 359 - ERROR - PUT-23729- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:11,817 - synapse.http.server - 112 - ERROR - PUT-23729- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:12,067 - synapse.federation.transport.server - 359 - ERROR - PUT-23731- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:12,071 - synapse.http.server - 112 - ERROR - PUT-23731- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:12,828 - synapse.federation.transport.server - 359 - ERROR - PUT-23732- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:12,832 - synapse.http.server - 112 - ERROR - PUT-23732- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:13,959 - synapse.metrics.background_process_metrics - 203 - ERROR - process_event_queue_for_federation-122- Background process 'process_event_queue_for_federation' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transaction_queue.py", line 160, in _process_event_queue_loop last_token = yield self.store.get_federation_out_pos("events") File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:14,047 - synapse.federation.federation_server - 227 - ERROR - PUT-23728-$15430897295407FYntV:berigora.net- Failed to handle PDU $15430897295407FYntV:berigora.net: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 216, in process_pdus_for_room origin, pdu File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 640, in _handle_received_pdu origin, pdu, sent_to_us_directly=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 262, in on_receive_pdu origin, pdu, prevs, min_depth File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 531, in _get_missing_events_for_pdu sent_to_us_directly=False, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 421, in on_receive_pdu auth_chain=auth_chain, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 604, in _process_received_pdu room = yield self.store.get_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:15,468 - synapse.push.pusherpool - 173 - ERROR - PUT-23733- Exception in pusher on_new_receipts Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/push/pusherpool.py", line 162, in on_new_receipts min_stream_id - 1, max_stream_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:15,472 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:15,473 - synapse.federation.transport.server - 359 - ERROR - PUT-23733- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:15,478 - synapse.http.server - 112 - ERROR - PUT-23733- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:15,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:15,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2324- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:17,053 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:17,055 - synapse.federation.transport.server - 359 - ERROR - PUT-23738- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:17,060 - synapse.http.server - 112 - ERROR - PUT-23738- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:17,957 - synapse.federation.transport.server - 359 - ERROR - PUT-23739- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:17,965 - synapse.http.server - 112 - ERROR - PUT-23739- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:18,146 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:18,149 - synapse.federation.transport.server - 359 - ERROR - PUT-23740- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:18,156 - synapse.http.server - 112 - ERROR - PUT-23740- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:18,586 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:18,589 - synapse.federation.transport.server - 359 - ERROR - PUT-23741- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:18,592 - synapse.http.server - 112 - ERROR - PUT-23741- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:20,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:20,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2325- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:20,819 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:20,822 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-196- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:20,877 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1179- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:21,801 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:21,803 - synapse.federation.transport.server - 359 - ERROR - PUT-23744- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:21,808 - synapse.http.server - 112 - ERROR - PUT-23744- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:22,194 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:22,197 - synapse.federation.transport.server - 359 - ERROR - PUT-23745- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:22,201 - synapse.http.server - 112 - ERROR - PUT-23745- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:23,468 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:23,471 - synapse.federation.transport.server - 359 - ERROR - PUT-23746- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:23,482 - synapse.http.server - 112 - ERROR - PUT-23746- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:24,226 - synapse.federation.transport.server - 359 - ERROR - PUT-23747- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:24,229 - synapse.http.server - 112 - ERROR - PUT-23747- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:25,502 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:25,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2326- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:25,601 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:25,605 - synapse.federation.transport.server - 359 - ERROR - PUT-23748- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:25,613 - synapse.http.server - 112 - ERROR - PUT-23748- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:26,518 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:26,521 - synapse.federation.transport.server - 359 - ERROR - PUT-23749- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:26,525 - synapse.http.server - 112 - ERROR - PUT-23749- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:28,928 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:28,932 - synapse.federation.transport.server - 359 - ERROR - PUT-23750- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:28,936 - synapse.http.server - 112 - ERROR - PUT-23750- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:29,891 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:29,894 - synapse.federation.transport.server - 359 - ERROR - PUT-23751- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:29,898 - synapse.http.server - 112 - ERROR - PUT-23751- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:30,553 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2327- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:31,005 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1180- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:33,288 - synapse.federation.transport.server - 359 - ERROR - PUT-23752- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:33,293 - synapse.http.server - 112 - ERROR - PUT-23752- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:33,366 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53619- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:33,592 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53447- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:34,514 - synapse.federation.transport.server - 359 - ERROR - PUT-23753- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:34,518 - synapse.http.server - 112 - ERROR - PUT-23753- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:35,059 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53602- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:35,159 - synapse.federation.transport.server - 359 - ERROR - PUT-23754- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:35,163 - synapse.http.server - 112 - ERROR - PUT-23754- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:35,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2328- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,175 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:02:36,178 - synapse.federation.transport.server - 359 - ERROR - PUT-23755- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,181 - synapse.http.server - 112 - ERROR - PUT-23755- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,466 - synapse.federation.transport.server - 359 - ERROR - PUT-23756- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,469 - synapse.http.server - 112 - ERROR - PUT-23756- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,707 - synapse.federation.transport.server - 359 - ERROR - PUT-23757- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,715 - synapse.http.server - 112 - ERROR - PUT-23757- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,830 - synapse.federation.transport.server - 359 - ERROR - PUT-23758- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:36,837 - synapse.http.server - 112 - ERROR - PUT-23758- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:37,724 - synapse.federation.transport.server - 359 - ERROR - PUT-23759- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:37,728 - synapse.http.server - 112 - ERROR - PUT-23759- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:40,353 - synapse.http.server - 112 - ERROR - POST-23760- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/pusher.py", line 112, in on_POST not_user_id=user.to_string() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/push/pusherpool.py", line 110, in remove_pushers_by_app_id_and_pushkey_not_user app_id, pushkey File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/pusher.py", line 97, in get_pushers_by ], desc="get_pushers_by" File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:40,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2329- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:40,629 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53145- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:41,004 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1181- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:44,081 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53004- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:45,231 - synapse.federation.transport.server - 359 - ERROR - PUT-23762- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:45,235 - synapse.http.server - 112 - ERROR - PUT-23762- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:45,637 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2330- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:45,652 - synapse.federation.transport.server - 359 - ERROR - PUT-23764- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:45,656 - synapse.http.server - 112 - ERROR - PUT-23764- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:50,005 - synapse.federation.transport.server - 359 - ERROR - PUT-23765- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:50,009 - synapse.http.server - 112 - ERROR - PUT-23765- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:50,580 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-52807- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:50,790 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2331- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:50,942 - synapse.federation.transport.server - 359 - ERROR - PUT-23766- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:50,947 - synapse.http.server - 112 - ERROR - PUT-23766- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:51,235 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1182- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:52,062 - synapse.federation.transport.server - 359 - ERROR - PUT-23767- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:52,068 - synapse.http.server - 112 - ERROR - PUT-23767- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:55,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2332- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:55,722 - synapse.federation.transport.server - 359 - ERROR - PUT-23768- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:55,728 - synapse.http.server - 112 - ERROR - PUT-23768- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:02:55,794 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53347- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:00,223 - synapse.federation.transport.server - 359 - ERROR - PUT-23769- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:00,226 - synapse.http.server - 112 - ERROR - PUT-23769- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:00,509 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2333- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:00,882 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1183- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:03,125 - synapse.federation.transport.server - 359 - ERROR - PUT-23770- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:03,131 - synapse.http.server - 112 - ERROR - PUT-23770- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:10,453 - synapse.http.server - 112 - ERROR - POST-23771- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/pusher.py", line 112, in on_POST not_user_id=user.to_string() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/push/pusherpool.py", line 110, in remove_pushers_by_app_id_and_pushkey_not_user app_id, pushkey File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/pusher.py", line 97, in get_pushers_by ], desc="get_pushers_by" File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:10,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:10,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2335- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:11,485 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1184- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:12,618 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53366- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:15,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2336- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:17,716 - synapse.federation.transport.server - 359 - ERROR - PUT-23774- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:17,722 - synapse.http.server - 112 - ERROR - PUT-23774- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:17,735 - synapse.federation.transport.server - 359 - ERROR - PUT-23775- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:17,739 - synapse.http.server - 112 - ERROR - PUT-23775- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:18,514 - synapse.federation.transport.server - 359 - ERROR - PUT-23776- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:18,520 - synapse.http.server - 112 - ERROR - PUT-23776- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:19,458 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:19,462 - synapse.federation.transport.server - 359 - ERROR - PUT-23777- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:19,467 - synapse.http.server - 112 - ERROR - PUT-23777- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:20,538 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2337- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:20,835 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:20,837 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-197- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:20,876 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1185- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:21,175 - synapse.federation.transport.server - 359 - ERROR - PUT-23778- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:21,178 - synapse.http.server - 112 - ERROR - PUT-23778- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:22,026 - synapse.federation.transport.server - 359 - ERROR - PUT-23779- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:22,029 - synapse.http.server - 112 - ERROR - PUT-23779- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:25,513 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2338- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:28,827 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-52936- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:29,294 - synapse.federation.transport.server - 359 - ERROR - PUT-23781- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:29,299 - synapse.http.server - 112 - ERROR - PUT-23781- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:29,536 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:29,539 - synapse.federation.transport.server - 359 - ERROR - PUT-23782- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:29,543 - synapse.http.server - 112 - ERROR - PUT-23782- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:30,604 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2339- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:30,902 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1186- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:31,681 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:31,684 - synapse.federation.transport.server - 359 - ERROR - PUT-23783- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:31,688 - synapse.http.server - 112 - ERROR - PUT-23783- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:33,760 - synapse.federation.transport.server - 359 - ERROR - PUT-23784- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:33,766 - synapse.http.server - 112 - ERROR - PUT-23784- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:34,045 - synapse.federation.transport.server - 359 - ERROR - PUT-23785- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:34,052 - synapse.http.server - 112 - ERROR - PUT-23785- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:34,131 - synapse.federation.transport.server - 359 - ERROR - PUT-23786- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:34,134 - synapse.http.server - 112 - ERROR - PUT-23786- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:36,222 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53337- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:39,342 - synapse.federation.transport.server - 359 - ERROR - PUT-23788- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:39,346 - synapse.http.server - 112 - ERROR - PUT-23788- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:41,897 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1187- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:42,067 - synapse.http.server - 112 - ERROR - POST-23789- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/pusher.py", line 126, in on_POST profile_tag=content.get('profile_tag', ""), File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/push/pusherpool.py", line 104, in add_pusher yield self.start_pusher_by_id(app_id, pushkey, user_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/push/pusherpool.py", line 182, in start_pusher_by_id app_id, pushkey File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/pusher.py", line 97, in get_pushers_by ], desc="get_pushers_by" File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:43,325 - synapse.federation.transport.server - 359 - ERROR - PUT-23790- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:43,329 - synapse.http.server - 112 - ERROR - PUT-23790- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:48,086 - synapse.federation.transport.server - 359 - ERROR - PUT-23794- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:48,092 - synapse.http.server - 112 - ERROR - PUT-23794- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:50,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2343- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:50,869 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:50,872 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1188- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:50,883 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:50,886 - synapse.federation.federation_server - 809 - ERROR - PUT-23795- Failed to handle edu 'm.device_list_update' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/device.py", line 485, in incoming_device_list_update yield self._handle_device_updates(user_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/device.py", line 536, in _handle_device_updates user_id, devices, stream_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:50,891 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:50,895 - synapse.federation.transport.server - 359 - ERROR - PUT-23795- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:50,901 - synapse.http.server - 112 - ERROR - PUT-23795- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:53,433 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:53,436 - synapse.federation.transport.server - 359 - ERROR - PUT-23796- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:53,440 - synapse.http.server - 112 - ERROR - PUT-23796- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:53,941 - synapse.federation.transport.server - 359 - ERROR - PUT-23797- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:53,945 - synapse.http.server - 112 - ERROR - PUT-23797- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:54,580 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:54,582 - synapse.federation.transport.server - 359 - ERROR - PUT-23798- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:54,586 - synapse.http.server - 112 - ERROR - PUT-23798- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:55,459 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:55,462 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53345- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:55,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:55,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2344- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:55,911 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:55,914 - synapse.federation.transport.server - 359 - ERROR - PUT-23799- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:55,919 - synapse.http.server - 112 - ERROR - PUT-23799- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:57,046 - synapse.federation.transport.server - 359 - ERROR - PUT-23800- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:57,049 - synapse.http.server - 112 - ERROR - PUT-23800- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:59,250 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:59,253 - synapse.federation.federation_server - 809 - ERROR - PUT-23801- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:59,266 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:59,269 - synapse.federation.transport.server - 359 - ERROR - PUT-23801- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:59,274 - synapse.http.server - 112 - ERROR - PUT-23801- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:59,747 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:03:59,750 - synapse.federation.transport.server - 359 - ERROR - PUT-23804- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:03:59,755 - synapse.http.server - 112 - ERROR - PUT-23804- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:00,896 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:00,899 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1189- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:01,540 - synapse.federation.transport.server - 359 - ERROR - PUT-23805- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:01,544 - synapse.http.server - 112 - ERROR - PUT-23805- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:05,633 - synapse.federation.transport.server - 359 - ERROR - PUT-23806- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:05,636 - synapse.http.server - 112 - ERROR - PUT-23806- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:07,669 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2346- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:08,010 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:08,013 - synapse.federation.transport.server - 359 - ERROR - PUT-23808- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:08,016 - synapse.http.server - 112 - ERROR - PUT-23808- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:08,122 - synapse.federation.transport.server - 359 - ERROR - PUT-23807- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:08,128 - synapse.http.server - 112 - ERROR - PUT-23807- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:10,513 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2347- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:10,690 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:10,693 - synapse.federation.transport.server - 359 - ERROR - PUT-23809- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:10,697 - synapse.http.server - 112 - ERROR - PUT-23809- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:10,884 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1190- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:14,952 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:14,955 - synapse.federation.transport.server - 359 - ERROR - PUT-23812- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:14,958 - synapse.http.server - 112 - ERROR - PUT-23812- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:15,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:15,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2348- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:18,157 - synapse.federation.transport.server - 359 - ERROR - PUT-23813- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:18,161 - synapse.http.server - 112 - ERROR - PUT-23813- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:18,928 - synapse.federation.transport.server - 359 - ERROR - PUT-23814- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:18,932 - synapse.http.server - 112 - ERROR - PUT-23814- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:19,308 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:19,311 - synapse.federation.transport.server - 359 - ERROR - PUT-23815- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:19,315 - synapse.http.server - 112 - ERROR - PUT-23815- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:20,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2349- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:20,861 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-198- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:20,874 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:20,877 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1191- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:20,883 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:20,885 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-198- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:24,120 - synapse.federation.transport.server - 359 - ERROR - PUT-23816- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:24,123 - synapse.http.server - 112 - ERROR - PUT-23816- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:24,704 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:24,707 - synapse.federation.transport.server - 359 - ERROR - PUT-23817- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:24,709 - synapse.http.server - 112 - ERROR - PUT-23817- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:25,506 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:25,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2350- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:30,505 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:30,509 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2351- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:30,874 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1192- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:35,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2352- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:35,557 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-53288- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:35,868 - synapse.federation.transport.server - 359 - ERROR - PUT-23820- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:35,873 - synapse.http.server - 112 - ERROR - PUT-23820- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:40,670 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2353- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:40,869 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:40,873 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1193- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:41,079 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:41,082 - synapse.federation.transport.server - 359 - ERROR - PUT-23821- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:41,089 - synapse.http.server - 112 - ERROR - PUT-23821- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:44,117 - synapse.federation.transport.server - 359 - ERROR - PUT-23824- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:44,122 - synapse.http.server - 112 - ERROR - PUT-23824- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:44,931 - synapse.federation.transport.server - 359 - ERROR - PUT-23825- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:44,935 - synapse.http.server - 112 - ERROR - PUT-23825- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:45,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:45,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2354- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:46,250 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:46,253 - synapse.federation.transport.server - 359 - ERROR - PUT-23826- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:46,258 - synapse.http.server - 112 - ERROR - PUT-23826- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:47,213 - synapse.federation.transport.server - 359 - ERROR - PUT-23827- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:47,218 - synapse.http.server - 112 - ERROR - PUT-23827- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:48,747 - synapse.federation.transport.server - 359 - ERROR - PUT-23828- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:48,752 - synapse.http.server - 112 - ERROR - PUT-23828- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:49,733 - synapse.federation.transport.server - 359 - ERROR - PUT-23829- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:49,737 - synapse.http.server - 112 - ERROR - PUT-23829- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:50,507 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:50,511 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2355- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:50,858 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:50,861 - synapse.handlers.presence - 277 - ERROR - - Error updating presence Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 274, in _update_states_and_catch_exception res = yield self._update_states(new_states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:50,962 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1194- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:52,082 - synapse.http.server - 112 - ERROR - PUT-23830- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:54,758 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:04:54,762 - synapse.federation.transport.server - 359 - ERROR - PUT-23831- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:54,766 - synapse.http.server - 112 - ERROR - PUT-23831- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:55,892 - synapse.federation.transport.server - 359 - ERROR - PUT-23832- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:04:55,894 - synapse.http.server - 112 - ERROR - PUT-23832- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:00,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2357- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:01,045 - synapse.http.server - 112 - ERROR - PUT-23835- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:01,105 - synapse.federation.transport.server - 359 - ERROR - PUT-23836- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:01,110 - synapse.http.server - 112 - ERROR - PUT-23836- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:02,050 - synapse.federation.transport.server - 359 - ERROR - PUT-23837- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:02,054 - synapse.http.server - 112 - ERROR - PUT-23837- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:03,375 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1195- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:03,393 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:03,395 - synapse.federation.transport.server - 359 - ERROR - PUT-23834- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:03,400 - synapse.http.server - 112 - ERROR - PUT-23834- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:03,879 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:03,883 - synapse.federation.transport.server - 359 - ERROR - PUT-23838- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:03,890 - synapse.http.server - 112 - ERROR - PUT-23838- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:04,990 - synapse.federation.transport.server - 359 - ERROR - PUT-23839- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:04,996 - synapse.http.server - 112 - ERROR - PUT-23839- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:05,030 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:05,033 - synapse.http.server - 112 - ERROR - PUT-23840- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:05,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:05,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2358- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:05,866 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:05,869 - synapse.federation.transport.server - 359 - ERROR - PUT-23841- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:05,871 - synapse.http.server - 112 - ERROR - PUT-23841- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:07,208 - synapse.federation.transport.server - 359 - ERROR - PUT-23842- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:07,213 - synapse.http.server - 112 - ERROR - PUT-23842- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:07,378 - synapse.federation.transport.server - 359 - ERROR - PUT-23843- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:07,381 - synapse.http.server - 112 - ERROR - PUT-23843- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:10,509 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2359- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:10,878 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1196- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:13,352 - synapse.federation.transport.server - 359 - ERROR - PUT-23845- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:13,359 - synapse.http.server - 112 - ERROR - PUT-23845- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:15,512 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:15,515 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2360- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:19,661 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:19,664 - synapse.federation.transport.server - 359 - ERROR - PUT-23849- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:19,668 - synapse.http.server - 112 - ERROR - PUT-23849- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:20,632 - synapse.metrics.background_process_metrics - 203 - ERROR - event_push_action_stream_orderings-19- Background process 'event_push_action_stream_orderings' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:20,861 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-199- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:20,924 - synapse.metrics.background_process_metrics - 203 - ERROR - generate_user_daily_visits-39- Background process 'generate_user_daily_visits' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:20,983 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-199- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:21,112 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1197- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:23,198 - synapse.federation.transport.server - 359 - ERROR - PUT-23851- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:23,208 - synapse.http.server - 112 - ERROR - PUT-23851- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:23,331 - synapse.federation.transport.server - 359 - ERROR - PUT-23852- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:23,336 - synapse.http.server - 112 - ERROR - PUT-23852- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:23,454 - synapse.federation.federation_server - 809 - ERROR - PUT-23850- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:25,564 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:05:25,566 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2361- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:28,390 - synapse.federation.transport.server - 359 - ERROR - PUT-23848- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:28,395 - synapse.http.server - 112 - ERROR - PUT-23848- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:30,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2362- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:31,874 - synapse.federation.transport.server - 359 - ERROR - PUT-23853- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:31,879 - synapse.http.server - 112 - ERROR - PUT-23853- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:32,046 - synapse.federation.transport.server - 359 - ERROR - PUT-23854- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:32,053 - synapse.http.server - 112 - ERROR - PUT-23854- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:35,281 - synapse.federation.transport.server - 359 - ERROR - PUT-23855- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:35,285 - synapse.http.server - 112 - ERROR - PUT-23855- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:40,251 - synapse.federation.transport.server - 359 - ERROR - PUT-23856- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:40,255 - synapse.http.server - 112 - ERROR - PUT-23856- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:40,582 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2364- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:45,491 - synapse.federation.transport.server - 359 - ERROR - PUT-23860- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:45,494 - synapse.http.server - 112 - ERROR - PUT-23860- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:46,316 - synapse.federation.transport.server - 359 - ERROR - PUT-23861- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:46,323 - synapse.http.server - 112 - ERROR - PUT-23861- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:46,680 - synapse.federation.transport.server - 359 - ERROR - PUT-23862- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:46,688 - synapse.http.server - 112 - ERROR - PUT-23862- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:47,658 - synapse.federation.transport.server - 359 - ERROR - PUT-23865- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:47,667 - synapse.http.server - 112 - ERROR - PUT-23865- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:49,381 - synapse.federation.transport.server - 359 - ERROR - PUT-23869- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:49,383 - synapse.http.server - 112 - ERROR - PUT-23869- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:50,443 - synapse.federation.transport.server - 359 - ERROR - PUT-23874- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:05:50,448 - synapse.http.server - 112 - ERROR - PUT-23874- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:09:55,057 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:10:09,538 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:11:20,597 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:13:15,369 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:15:41,468 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:15:49,809 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:16:54,920 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:16:56,671 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:16:56,676 - synapse.federation.transport.server - 359 - ERROR - PUT-24977- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:16:56,681 - synapse.http.server - 112 - ERROR - PUT-24977- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:16:56,948 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:16:56,951 - synapse.federation.transport.server - 359 - ERROR - PUT-24978- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:16:56,954 - synapse.http.server - 112 - ERROR - PUT-24978- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:00,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:00,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2496- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:00,869 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:00,871 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1267- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:00,967 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:00,971 - synapse.federation.transport.server - 359 - ERROR - PUT-24979- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:00,975 - synapse.http.server - 112 - ERROR - PUT-24979- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:03,858 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:03,862 - synapse.federation.transport.server - 359 - ERROR - PUT-24980- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:03,867 - synapse.http.server - 112 - ERROR - PUT-24980- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:05,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:05,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2497- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:05,671 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:05,673 - synapse.federation.transport.server - 359 - ERROR - PUT-24981- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:05,678 - synapse.http.server - 112 - ERROR - PUT-24981- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:07,028 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:07,031 - synapse.federation.transport.server - 359 - ERROR - PUT-24982- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:07,036 - synapse.http.server - 112 - ERROR - PUT-24982- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:10,499 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:10,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2498- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:11,176 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:11,179 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1268- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:12,007 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:12,010 - synapse.federation.federation_server - 809 - ERROR - PUT-24983- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:12,094 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:12,097 - synapse.federation.transport.server - 359 - ERROR - PUT-24983- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:12,103 - synapse.http.server - 112 - ERROR - PUT-24983- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:16,339 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:16,342 - synapse.federation.transport.server - 359 - ERROR - PUT-24985- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:16,348 - synapse.http.server - 112 - ERROR - PUT-24985- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:16,606 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:17:16,609 - synapse.federation.transport.server - 359 - ERROR - PUT-24984- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:16,613 - synapse.http.server - 112 - ERROR - PUT-24984- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:18,150 - synapse.federation.transport.server - 359 - ERROR - PUT-24992- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:18,155 - synapse.http.server - 112 - ERROR - PUT-24992- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:19,999 - synapse.federation.transport.server - 359 - ERROR - PUT-24998- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:17:20,003 - synapse.http.server - 112 - ERROR - PUT-24998- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,007 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,011 - synapse.federation.transport.server - 359 - ERROR - PUT-25143- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,016 - synapse.http.server - 112 - ERROR - PUT-25143- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,093 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,096 - synapse.federation.transport.server - 359 - ERROR - PUT-25146- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,100 - synapse.http.server - 112 - ERROR - PUT-25146- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,149 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,154 - synapse.federation.transaction_queue - 319 - ERROR - user_joined_room-48- Error sending presence states to servers Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transaction_queue.py", line 317, in send_presence yield self._process_presence_inner(list(states_map.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transaction_queue.py", line 332, in _process_presence_inner hosts_and_states = yield get_interested_remotes(self.store, states, self.state) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 1331, in get_interested_remotes hosts = yield state_handler.get_current_hosts_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 177, in get_current_hosts_in_room joined_hosts = yield self.store.get_joined_hosts(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 521, in _get_joined_hosts joined_hosts = yield cache.get_destinations(state_entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 792, in get_destinations self.room_id, state_entry, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,189 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,194 - synapse.federation.transport.server - 359 - ERROR - PUT-25142- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,203 - synapse.http.server - 112 - ERROR - PUT-25142- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,318 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,321 - synapse.federation.transport.server - 359 - ERROR - PUT-25145- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,326 - synapse.http.server - 112 - ERROR - PUT-25145- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,448 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,451 - synapse.federation.federation_server - 809 - ERROR - PUT-25138- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,490 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,492 - synapse.federation.transport.server - 359 - ERROR - PUT-25151- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,496 - synapse.http.server - 112 - ERROR - PUT-25151- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,570 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,574 - synapse.federation.federation_server - 809 - ERROR - PUT-25129- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 603, in _persist_and_notify parties = yield get_interested_parties(self.store, states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 1293, in get_interested_parties room_ids = yield store.get_rooms_for_user(state.user_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 279, in get_rooms_for_user user_id, on_invalidate=on_invalidate, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 267, in get_rooms_for_user_with_stream_ordering user_id, membership_list=[Membership.JOIN], File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,592 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,595 - synapse.federation.transport.server - 359 - ERROR - PUT-25152- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,600 - synapse.http.server - 112 - ERROR - PUT-25152- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,618 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,621 - synapse.federation.transport.server - 359 - ERROR - PUT-25153- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,626 - synapse.http.server - 112 - ERROR - PUT-25153- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,664 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,667 - synapse.federation.transport.server - 359 - ERROR - PUT-25149- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,672 - synapse.http.server - 112 - ERROR - PUT-25149- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,693 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,697 - synapse.federation.transport.server - 359 - ERROR - PUT-25154- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,701 - synapse.http.server - 112 - ERROR - PUT-25154- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,733 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,737 - synapse.federation.transport.server - 359 - ERROR - PUT-25155- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,741 - synapse.http.server - 112 - ERROR - PUT-25155- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,803 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,808 - synapse.http.server - 112 - ERROR - PUT-25147- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:18,855 - synapse.federation.transport.server - 359 - ERROR - PUT-25156- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,862 - synapse.http.server - 112 - ERROR - PUT-25156- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,915 - synapse.federation.transport.server - 359 - ERROR - PUT-25158- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,918 - synapse.http.server - 112 - ERROR - PUT-25158- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,987 - synapse.federation.transport.server - 359 - ERROR - PUT-25161- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:18,992 - synapse.http.server - 112 - ERROR - PUT-25161- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,014 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,023 - synapse.federation.transport.server - 359 - ERROR - PUT-25148- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,030 - synapse.http.server - 112 - ERROR - PUT-25148- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,158 - synapse.federation.transport.server - 359 - ERROR - PUT-25163- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,160 - synapse.http.server - 112 - ERROR - PUT-25163- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,277 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,280 - synapse.federation.transport.server - 359 - ERROR - PUT-25150- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,284 - synapse.http.server - 112 - ERROR - PUT-25150- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,469 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,472 - synapse.federation.transport.server - 359 - ERROR - PUT-25138- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,476 - synapse.http.server - 112 - ERROR - PUT-25138- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,614 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,616 - synapse.federation.transport.server - 359 - ERROR - PUT-25159- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,619 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,623 - synapse.http.server - 112 - ERROR - PUT-25159- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,631 - synapse.federation.transport.server - 359 - ERROR - PUT-25165- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,636 - synapse.http.server - 112 - ERROR - PUT-25165- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,639 - synapse.federation.transport.server - 359 - ERROR - PUT-25160- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,643 - synapse.http.server - 112 - ERROR - PUT-25160- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,648 - synapse.federation.transport.server - 359 - ERROR - PUT-25162- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,652 - synapse.http.server - 112 - ERROR - PUT-25162- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,666 - synapse.federation.transport.server - 359 - ERROR - PUT-25157- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,672 - synapse.http.server - 112 - ERROR - PUT-25157- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,676 - synapse.federation.transport.server - 359 - ERROR - PUT-25164- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,682 - synapse.http.server - 112 - ERROR - PUT-25164- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,700 - synapse.federation.transport.server - 359 - ERROR - PUT-25166- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,705 - synapse.http.server - 112 - ERROR - PUT-25166- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,712 - synapse.federation.transport.server - 359 - ERROR - PUT-25167- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,717 - synapse.http.server - 112 - ERROR - PUT-25167- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,780 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:18:19,783 - synapse.federation.transport.server - 359 - ERROR - PUT-25129- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:19,787 - synapse.http.server - 112 - ERROR - PUT-25129- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,186 - synapse.federation.transport.server - 359 - ERROR - PUT-25168- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,190 - synapse.http.server - 112 - ERROR - PUT-25168- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,264 - synapse.federation.transport.server - 359 - ERROR - PUT-25169- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,269 - synapse.http.server - 112 - ERROR - PUT-25169- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2512- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,588 - synapse.federation.transport.server - 359 - ERROR - PUT-25170- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,591 - synapse.http.server - 112 - ERROR - PUT-25170- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,790 - synapse.federation.transport.server - 359 - ERROR - PUT-25171- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,794 - synapse.http.server - 112 - ERROR - PUT-25171- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,812 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55365- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,824 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-212- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,861 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-212- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:20,876 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1275- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,125 - synapse.federation.transport.server - 359 - ERROR - PUT-25172- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,129 - synapse.http.server - 112 - ERROR - PUT-25172- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,255 - synapse.federation.transport.server - 359 - ERROR - PUT-25173- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,258 - synapse.http.server - 112 - ERROR - PUT-25173- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,489 - synapse.federation.transport.server - 359 - ERROR - PUT-25174- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,495 - synapse.http.server - 112 - ERROR - PUT-25174- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,836 - synapse.federation.transport.server - 359 - ERROR - PUT-25175- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:21,844 - synapse.http.server - 112 - ERROR - PUT-25175- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,095 - synapse.federation.transport.server - 359 - ERROR - PUT-25176- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,099 - synapse.http.server - 112 - ERROR - PUT-25176- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,198 - synapse.federation.transport.server - 359 - ERROR - PUT-25177- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,202 - synapse.http.server - 112 - ERROR - PUT-25177- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,261 - synapse.http.server - 112 - ERROR - PUT-25178- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,279 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55244- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,372 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55242- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,473 - synapse.federation.transport.server - 359 - ERROR - PUT-25179- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,478 - synapse.http.server - 112 - ERROR - PUT-25179- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,745 - synapse.federation.transport.server - 359 - ERROR - PUT-25180- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,750 - synapse.http.server - 112 - ERROR - PUT-25180- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,760 - synapse.federation.transport.server - 359 - ERROR - PUT-25181- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:22,764 - synapse.http.server - 112 - ERROR - PUT-25181- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:23,409 - synapse.federation.transport.server - 359 - ERROR - PUT-25182- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:23,415 - synapse.http.server - 112 - ERROR - PUT-25182- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:23,655 - synapse.federation.transport.server - 359 - ERROR - PUT-25183- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:23,660 - synapse.http.server - 112 - ERROR - PUT-25183- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:23,954 - synapse.federation.transport.server - 359 - ERROR - PUT-25184- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:23,958 - synapse.http.server - 112 - ERROR - PUT-25184- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:24,021 - synapse.federation.transport.server - 359 - ERROR - PUT-25185- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:24,025 - synapse.http.server - 112 - ERROR - PUT-25185- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:24,298 - synapse.federation.transport.server - 359 - ERROR - PUT-25186- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:24,303 - synapse.http.server - 112 - ERROR - PUT-25186- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:24,340 - synapse.federation.transport.server - 359 - ERROR - PUT-25187- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:24,344 - synapse.http.server - 112 - ERROR - PUT-25187- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:24,548 - synapse.http.server - 112 - ERROR - PUT-25188- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,104 - synapse.federation.transport.server - 359 - ERROR - PUT-25189- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,109 - synapse.http.server - 112 - ERROR - PUT-25189- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,174 - synapse.federation.transport.server - 359 - ERROR - PUT-25190- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,179 - synapse.http.server - 112 - ERROR - PUT-25190- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,220 - synapse.crypto.keyring - 688 - ERROR - PUT-25191- Got Exception when downloading keys for matrix.cluster.terracrypt.net: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,494 - synapse.federation.transport.server - 359 - ERROR - PUT-25192- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,500 - synapse.http.server - 112 - ERROR - PUT-25192- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,514 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2513- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,584 - synapse.federation.transport.server - 359 - ERROR - PUT-25193- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,587 - synapse.http.server - 112 - ERROR - PUT-25193- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:25,764 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55100- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:26,097 - synapse.federation.transport.server - 359 - ERROR - PUT-25194- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:26,100 - synapse.http.server - 112 - ERROR - PUT-25194- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:26,193 - synapse.federation.transport.server - 359 - ERROR - PUT-25195- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:26,196 - synapse.http.server - 112 - ERROR - PUT-25195- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:26,874 - synapse.federation.transport.server - 359 - ERROR - PUT-25196- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:26,880 - synapse.http.server - 112 - ERROR - PUT-25196- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:27,233 - synapse.federation.transport.server - 359 - ERROR - PUT-25197- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:27,236 - synapse.http.server - 112 - ERROR - PUT-25197- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:27,487 - synapse.federation.transport.server - 359 - ERROR - PUT-25198- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:27,496 - synapse.http.server - 112 - ERROR - PUT-25198- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,177 - synapse.federation.transport.server - 359 - ERROR - PUT-25199- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,181 - synapse.http.server - 112 - ERROR - PUT-25199- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,216 - synapse.federation.transport.server - 359 - ERROR - PUT-25200- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,219 - synapse.http.server - 112 - ERROR - PUT-25200- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,482 - synapse.federation.transport.server - 359 - ERROR - PUT-25201- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,485 - synapse.http.server - 112 - ERROR - PUT-25201- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,978 - synapse.federation.transport.server - 359 - ERROR - PUT-25202- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:28,983 - synapse.http.server - 112 - ERROR - PUT-25202- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:29,398 - synapse.crypto.keyring - 688 - ERROR - PUT-25203- Got Exception when downloading keys for matrix.notice.at: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:29,674 - synapse.federation.transport.server - 359 - ERROR - PUT-25204- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:29,678 - synapse.http.server - 112 - ERROR - PUT-25204- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:29,707 - synapse.federation.transport.server - 359 - ERROR - PUT-25205- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:29,715 - synapse.http.server - 112 - ERROR - PUT-25205- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:29,792 - synapse.federation.transport.server - 359 - ERROR - PUT-25206- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:29,796 - synapse.http.server - 112 - ERROR - PUT-25206- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:30,545 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2514- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:30,607 - synapse.federation.transport.server - 359 - ERROR - PUT-25207- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:30,611 - synapse.http.server - 112 - ERROR - PUT-25207- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:30,872 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1276- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:32,441 - synapse.crypto.keyring - 688 - ERROR - PUT-25208- Got Exception when downloading keys for schefczyk.net: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,060 - synapse.federation.transport.server - 359 - ERROR - PUT-25209- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,066 - synapse.http.server - 112 - ERROR - PUT-25209- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,073 - synapse.federation.transport.server - 359 - ERROR - PUT-25210- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,080 - synapse.http.server - 112 - ERROR - PUT-25210- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,322 - synapse.federation.transport.server - 359 - ERROR - PUT-25211- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,327 - synapse.http.server - 112 - ERROR - PUT-25211- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,455 - synapse.federation.transport.server - 359 - ERROR - PUT-25213- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,459 - synapse.http.server - 112 - ERROR - PUT-25213- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,489 - synapse.federation.transport.server - 359 - ERROR - PUT-25212- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:33,495 - synapse.http.server - 112 - ERROR - PUT-25212- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:34,801 - synapse.http.server - 112 - ERROR - PUT-25214- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:35,564 - synapse.federation.transport.server - 359 - ERROR - PUT-25216- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:35,568 - synapse.http.server - 112 - ERROR - PUT-25216- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:35,583 - synapse.crypto.keyring - 688 - ERROR - PUT-25215- Got Exception when downloading keys for naszademokracja.pl: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:35,742 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2515- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:35,967 - synapse.federation.transport.server - 359 - ERROR - PUT-25217- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:35,970 - synapse.http.server - 112 - ERROR - PUT-25217- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:36,693 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55381- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:37,083 - synapse.federation.transport.server - 359 - ERROR - PUT-25218- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:37,089 - synapse.http.server - 112 - ERROR - PUT-25218- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:37,479 - synapse.federation.transport.server - 359 - ERROR - PUT-25219- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:37,484 - synapse.http.server - 112 - ERROR - PUT-25219- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:38,454 - synapse.federation.transport.server - 359 - ERROR - PUT-25220- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:38,459 - synapse.http.server - 112 - ERROR - PUT-25220- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,304 - synapse.federation.transport.server - 359 - ERROR - PUT-25222- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,306 - synapse.http.server - 112 - ERROR - PUT-25222- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,399 - synapse.federation.transport.server - 359 - ERROR - PUT-25221- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,403 - synapse.http.server - 112 - ERROR - PUT-25221- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2516- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,506 - synapse.federation.transport.server - 359 - ERROR - PUT-25223- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,509 - synapse.http.server - 112 - ERROR - PUT-25223- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,843 - synapse.federation.transport.server - 359 - ERROR - PUT-25224- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,846 - synapse.http.server - 112 - ERROR - PUT-25224- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:40,875 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1277- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:41,477 - synapse.federation.transport.server - 359 - ERROR - PUT-25225- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:41,480 - synapse.http.server - 112 - ERROR - PUT-25225- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:45,396 - synapse.crypto.keyring - 688 - ERROR - PUT-25226- Got Exception when downloading keys for matrix.cosmolab.world: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:45,654 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2517- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:46,558 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55512- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:46,765 - synapse.federation.transport.server - 359 - ERROR - PUT-25229- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:46,773 - synapse.http.server - 112 - ERROR - PUT-25229- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:47,185 - synapse.federation.transport.server - 359 - ERROR - PUT-25230- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:47,189 - synapse.http.server - 112 - ERROR - PUT-25230- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:47,589 - synapse.federation.transport.server - 359 - ERROR - PUT-25231- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:47,596 - synapse.http.server - 112 - ERROR - PUT-25231- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:50,663 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2518- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:50,969 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1278- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:51,728 - synapse.federation.transport.server - 359 - ERROR - PUT-25232- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:51,735 - synapse.http.server - 112 - ERROR - PUT-25232- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:52,392 - synapse.http.server - 112 - ERROR - PUT-25233- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:53,052 - synapse.federation.transport.server - 359 - ERROR - PUT-25234- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:53,055 - synapse.http.server - 112 - ERROR - PUT-25234- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:55,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2519- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:56,445 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-56019- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:57,277 - synapse.federation.transport.server - 359 - ERROR - PUT-25235- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:57,281 - synapse.http.server - 112 - ERROR - PUT-25235- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:57,614 - synapse.federation.transport.server - 359 - ERROR - PUT-25236- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:57,618 - synapse.http.server - 112 - ERROR - PUT-25236- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:58,342 - synapse.federation.transport.server - 359 - ERROR - PUT-25237- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:18:58,344 - synapse.http.server - 112 - ERROR - PUT-25237- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:00,322 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55246- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:00,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2520- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:00,888 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1279- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:02,970 - synapse.federation.transport.server - 359 - ERROR - PUT-25238- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:02,974 - synapse.http.server - 112 - ERROR - PUT-25238- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:05,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2521- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:05,603 - synapse.federation.transport.server - 359 - ERROR - PUT-25239- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:05,607 - synapse.http.server - 112 - ERROR - PUT-25239- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:05,654 - synapse.federation.transport.server - 359 - ERROR - PUT-25240- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:05,658 - synapse.http.server - 112 - ERROR - PUT-25240- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:06,167 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-54705- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:07,415 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55386- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:07,424 - synapse.federation.transport.server - 359 - ERROR - PUT-25241- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:07,429 - synapse.http.server - 112 - ERROR - PUT-25241- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:07,774 - synapse.federation.transport.server - 359 - ERROR - PUT-25242- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:07,779 - synapse.http.server - 112 - ERROR - PUT-25242- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:08,718 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-54782- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:10,510 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2522- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:10,879 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1280- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:12,800 - synapse.federation.transport.server - 359 - ERROR - PUT-25243- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:12,803 - synapse.http.server - 112 - ERROR - PUT-25243- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:14,954 - synapse.federation.transport.server - 359 - ERROR - PUT-25244- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:14,960 - synapse.http.server - 112 - ERROR - PUT-25244- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:15,570 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2523- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:17,966 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55504- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:20,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2524- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:20,825 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-213- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:20,881 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1281- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:21,567 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55045- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:25,647 - synapse.federation.transport.server - 359 - ERROR - PUT-25247- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:25,651 - synapse.http.server - 112 - ERROR - PUT-25247- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:27,651 - synapse.federation.transport.server - 359 - ERROR - PUT-25248- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:27,655 - synapse.http.server - 112 - ERROR - PUT-25248- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:30,499 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:30,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2526- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:30,878 - synapse.federation.transport.server - 359 - ERROR - PUT-25249- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:30,883 - synapse.http.server - 112 - ERROR - PUT-25249- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:30,889 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1282- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:32,128 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55375- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:32,694 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:32,697 - synapse.federation.transport.server - 359 - ERROR - PUT-25250- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:32,702 - synapse.http.server - 112 - ERROR - PUT-25250- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:35,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2527- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:37,083 - synapse.federation.transport.server - 359 - ERROR - PUT-25251- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:37,088 - synapse.http.server - 112 - ERROR - PUT-25251- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:39,077 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:39,079 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55265- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:40,488 - synapse.federation.transport.server - 359 - ERROR - PUT-25252- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:40,492 - synapse.http.server - 112 - ERROR - PUT-25252- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:40,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2528- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:40,876 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:40,879 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1283- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:42,379 - synapse.federation.transport.server - 359 - ERROR - PUT-25253- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:42,382 - synapse.http.server - 112 - ERROR - PUT-25253- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:42,788 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-54762- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:43,102 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-54830- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:45,505 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:45,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2529- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:47,154 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-54851- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:47,470 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55348- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:48,672 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:19:48,676 - synapse.federation.transport.server - 359 - ERROR - PUT-25256- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:48,681 - synapse.http.server - 112 - ERROR - PUT-25256- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:50,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2530- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:50,898 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1284- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:54,846 - synapse.federation.transport.server - 359 - ERROR - PUT-25257- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:54,849 - synapse.http.server - 112 - ERROR - PUT-25257- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:55,657 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2531- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:55,813 - synapse.federation.transport.server - 359 - ERROR - PUT-25258- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:55,817 - synapse.http.server - 112 - ERROR - PUT-25258- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:57,813 - synapse.federation.transport.server - 359 - ERROR - PUT-25259- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:19:57,817 - synapse.http.server - 112 - ERROR - PUT-25259- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:00,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2532- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:00,878 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1285- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:03,183 - synapse.federation.transport.server - 359 - ERROR - PUT-25260- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:03,185 - synapse.http.server - 112 - ERROR - PUT-25260- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:03,714 - synapse.federation.transport.server - 359 - ERROR - PUT-25261- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:03,719 - synapse.http.server - 112 - ERROR - PUT-25261- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:04,525 - synapse.federation.transport.server - 359 - ERROR - PUT-25262- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:04,531 - synapse.http.server - 112 - ERROR - PUT-25262- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:04,788 - synapse.federation.transport.server - 359 - ERROR - PUT-25263- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:04,793 - synapse.http.server - 112 - ERROR - PUT-25263- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:05,824 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2533- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:10,770 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2534- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:10,874 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1286- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:11,140 - synapse.federation.transport.server - 359 - ERROR - PUT-25265- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:11,144 - synapse.http.server - 112 - ERROR - PUT-25265- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:11,964 - synapse.federation.transport.server - 359 - ERROR - PUT-25266- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:11,969 - synapse.http.server - 112 - ERROR - PUT-25266- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:15,672 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2535- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:16,840 - synapse.federation.transport.server - 359 - ERROR - PUT-25269- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:16,845 - synapse.http.server - 112 - ERROR - PUT-25269- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:20,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2536- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:20,828 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-214- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:20,862 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-214- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:20,877 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1287- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:20,923 - synapse.metrics.background_process_metrics - 203 - ERROR - generate_user_daily_visits-42- Background process 'generate_user_daily_visits' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:23,480 - synapse.federation.transport.server - 359 - ERROR - PUT-25270- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:23,485 - synapse.http.server - 112 - ERROR - PUT-25270- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:24,844 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55111- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:25,516 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2537- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:28,668 - synapse.federation.transport.server - 359 - ERROR - PUT-25271- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:28,672 - synapse.http.server - 112 - ERROR - PUT-25271- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:29,605 - synapse.federation.transport.server - 359 - ERROR - PUT-25272- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:29,610 - synapse.http.server - 112 - ERROR - PUT-25272- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:30,352 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-57222- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:30,774 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2538- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:30,874 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1288- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:31,158 - synapse.http.server - 112 - ERROR - GET-25273- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/devices.py", line 116, in on_GET device_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/device.py", line 139, in get_device device = yield self.store.get_device(user_id, device_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:31,291 - synapse.http.server - 112 - ERROR - POST-25275- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/keys.py", line 94, in on_POST user_id, device_id, body File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/e2e_keys.py", line 277, in upload_keys_for_user user_id, device_id, time_now, device_keys, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:34,262 - synapse.federation.transport.server - 359 - ERROR - PUT-25276- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:34,268 - synapse.http.server - 112 - ERROR - PUT-25276- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:35,434 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55188- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:35,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2539- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:36,088 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55097- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:36,425 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55330- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:38,960 - synapse.federation.transport.server - 359 - ERROR - PUT-25277- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:38,962 - synapse.http.server - 112 - ERROR - PUT-25277- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:40,897 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1289- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:41,168 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2540- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:46,061 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2541- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:46,065 - synapse.util.retryutils - 205 - ERROR - federation_transaction_transmission_loop-55491- Failed to store destination_retry_timings Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 201, in store_retry_timings self.destination, retry_last_ts, self.retry_interval File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:50,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2542- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:50,877 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1290- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:54,062 - synapse.federation.transport.server - 359 - ERROR - PUT-25280- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:54,065 - synapse.http.server - 112 - ERROR - PUT-25280- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:55,188 - synapse.federation.transport.server - 359 - ERROR - PUT-25281- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:55,192 - synapse.http.server - 112 - ERROR - PUT-25281- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:56,197 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2543- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:57,311 - synapse.federation.transport.server - 359 - ERROR - PUT-25283- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:20:57,315 - synapse.http.server - 112 - ERROR - PUT-25283- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:00,136 - synapse.http.server - 112 - ERROR - PUT-25284- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:00,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2544- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:01,191 - synapse.federation.transport.server - 359 - ERROR - PUT-25285- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:01,195 - synapse.http.server - 112 - ERROR - PUT-25285- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:01,766 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1291- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:02,397 - synapse.federation.transport.server - 359 - ERROR - PUT-25286- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:02,401 - synapse.http.server - 112 - ERROR - PUT-25286- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:07,928 - synapse.federation.transaction_queue - 319 - ERROR - PUT-25287- Error sending presence states to servers Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transaction_queue.py", line 317, in send_presence yield self._process_presence_inner(list(states_map.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transaction_queue.py", line 332, in _process_presence_inner hosts_and_states = yield get_interested_remotes(self.store, states, self.state) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 1331, in get_interested_remotes hosts = yield state_handler.get_current_hosts_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 177, in get_current_hosts_in_room joined_hosts = yield self.store.get_joined_hosts(room_id, entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 521, in _get_joined_hosts joined_hosts = yield cache.get_destinations(state_entry) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 792, in get_destinations self.room_id, state_entry, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/roommember.py", line 405, in _get_joined_users_from_context desc="_get_joined_users_from_context", File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 748, in _simple_select_many_batch table, column, chunk, keyvalues, retcols File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:07,934 - synapse.federation.transport.server - 359 - ERROR - PUT-25287- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:07,940 - synapse.http.server - 112 - ERROR - PUT-25287- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 252, in _handle_incoming_transaction 200, response File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:22,293 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:21:22,296 - synapse.federation.federation_server - 809 - ERROR - PUT-25294- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:21:29,215 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:21:29,217 - synapse.federation.transport.server - 359 - ERROR - PUT-25300- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:29,221 - synapse.http.server - 112 - ERROR - PUT-25300- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:32,526 - synapse.http.server - 112 - ERROR - GET-25301- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 167, in on_GET full_state=full_state File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 239, in wait_for_sync_for_user sync_config, since_token, timeout, full_state, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 261, in _wait_for_sync_for_user sync_config, since_token, full_state=full_state, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 929, in generate_sync_result sync_result_builder, account_data_by_room File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1298, in _generate_sync_entry_for_rooms yield concurrently_execute(handle_room_entries, room_entries, 10) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/async_helpers.py", line 149, in _concurrently_execute_inner yield func(next(it)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1682, in _generate_room_entry full_state=full_state File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 791, in compute_state_delta state_filter=state_filter, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 511, in get_state_at room_id, end_token=stream_position.room_key, limit=1, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py", line 358, in get_recent_events_for_room room_id, limit, end_token, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py", line 395, in get_recent_event_ids_for_room room_id, from_token=end_token, limit=limit, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:21:42,653 - synapse.http.server - 112 - ERROR - GET-25303- Failed handle request via : : Traceback (most recent call last): --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/sync.py", line 167, in on_GET full_state=full_state File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 239, in wait_for_sync_for_user sync_config, since_token, timeout, full_state, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 269, in _wait_for_sync_for_user from_token=since_token, File "/usr/local/lib/python3.5/dist-packages/synapse/notifier.py", line 350, in wait_for_events result = yield callback(prev_token, current_token) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 929, in generate_sync_result sync_result_builder, account_data_by_room File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1274, in _generate_sync_entry_for_rooms res = yield self._get_rooms_changed(sync_result_builder, ignored_users) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/sync.py", line 1493, in _get_rooms_changed limit=timeline_limit + 1, File "/usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py", line 212, in get_room_events_stream_for_rooms ], consumeErrors=True)) twisted.internet.defer.FirstError: FirstError[#3, [Failure instance: Traceback: : server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:501:errback /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:568:_startRunCallbacks /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:654:_runCallbacks /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1475:gotResult --- --- /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1416:_inlineCallbacks /usr/local/lib/python3.5/dist-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator /usr/local/lib/python3.5/dist-packages/synapse/storage/stream.py:279:get_room_events_stream_for_room /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1416:_inlineCallbacks /usr/local/lib/python3.5/dist-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator /usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py:325:runInteraction /usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py:1416:_inlineCallbacks /usr/local/lib/python3.5/dist-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator /usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py:374:runWithConnection /usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py:250:inContext /usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py:266: /usr/local/lib/python3.5/dist-packages/twisted/python/context.py:122:callWithContext /usr/local/lib/python3.5/dist-packages/twisted/python/context.py:85:callWithContext /usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py:295:_runWithConnection /usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py:36:__init__ /usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py:76:reconnect /usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py:429:connect /usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py:130:connect ]] 2018-11-24 20:26:42,481 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:27:12,032 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:28:15,870 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:30:05,497 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:05,499 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2651- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:06,356 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:06,360 - synapse.federation.transport.server - 359 - ERROR - PUT-25911- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:06,367 - synapse.http.server - 112 - ERROR - PUT-25911- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,161 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,165 - synapse.federation.transport.server - 359 - ERROR - PUT-25912- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,171 - synapse.http.server - 112 - ERROR - PUT-25912- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,238 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,242 - synapse.federation.transport.server - 359 - ERROR - PUT-25913- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,251 - synapse.http.server - 112 - ERROR - PUT-25913- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,918 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,921 - synapse.federation.transport.server - 359 - ERROR - PUT-25914- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:07,926 - synapse.http.server - 112 - ERROR - PUT-25914- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,424 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,428 - synapse.federation.transport.server - 359 - ERROR - PUT-25916- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,433 - synapse.http.server - 112 - ERROR - PUT-25916- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,770 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,773 - synapse.federation.transport.server - 359 - ERROR - PUT-25917- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,778 - synapse.http.server - 112 - ERROR - PUT-25917- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,921 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,924 - synapse.federation.transport.server - 359 - ERROR - PUT-25918- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,927 - synapse.http.server - 112 - ERROR - PUT-25918- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,994 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:08,997 - synapse.federation.transport.server - 359 - ERROR - PUT-25919- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,002 - synapse.http.server - 112 - ERROR - PUT-25919- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,030 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,033 - synapse.federation.transport.server - 359 - ERROR - PUT-25920- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,037 - synapse.http.server - 112 - ERROR - PUT-25920- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,056 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,060 - synapse.federation.transport.server - 359 - ERROR - PUT-25921- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,067 - synapse.http.server - 112 - ERROR - PUT-25921- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,267 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,270 - synapse.federation.transport.server - 359 - ERROR - PUT-25922- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,276 - synapse.http.server - 112 - ERROR - PUT-25922- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,672 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,675 - synapse.federation.transport.server - 359 - ERROR - PUT-25924- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,680 - synapse.http.server - 112 - ERROR - PUT-25924- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,716 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,723 - synapse.federation.transport.server - 359 - ERROR - PUT-25926- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,730 - synapse.http.server - 112 - ERROR - PUT-25926- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,835 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,841 - synapse.federation.transport.server - 359 - ERROR - PUT-25929- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,847 - synapse.http.server - 112 - ERROR - PUT-25929- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:09,867 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:09,872 - synapse.crypto.keyring - 688 - ERROR - PUT-25930- Got Exception when downloading keys for jones.ninja: OperationalError server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 674, in _handle_key_deferred _, key_id, verify_key = yield verify_request.deferred File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 283, in do_iterations results = yield fn(missing_keys.items()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/crypto/keyring.py", line 358, in get_keys_from_store ).addErrback(unwrapFirstError)) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 121, in get_server_verify_keys key = yield self._get_server_verify_key(server_name, key_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/keys.py", line 100, in _get_server_verify_key allow_none=True, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,045 - synapse.federation.transport.server - 359 - ERROR - PUT-25933- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,050 - synapse.http.server - 112 - ERROR - PUT-25933- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,163 - synapse.federation.transport.server - 359 - ERROR - PUT-25934- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,168 - synapse.http.server - 112 - ERROR - PUT-25934- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,171 - synapse.federation.transport.server - 359 - ERROR - PUT-25935- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,177 - synapse.http.server - 112 - ERROR - PUT-25935- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,225 - synapse.federation.transport.server - 359 - ERROR - PUT-25937- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,232 - synapse.http.server - 112 - ERROR - PUT-25937- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,282 - synapse.federation.transport.server - 359 - ERROR - PUT-25939- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,286 - synapse.http.server - 112 - ERROR - PUT-25939- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,347 - synapse.federation.transport.server - 359 - ERROR - PUT-25940- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:10,352 - synapse.http.server - 112 - ERROR - PUT-25940- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:30:34,644 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:30:56,893 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:56,896 - synapse.federation.transport.server - 359 - ERROR - PUT-26388- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:56,900 - synapse.http.server - 112 - ERROR - PUT-26388- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:57,107 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:57,110 - synapse.federation.transport.server - 359 - ERROR - PUT-26389- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:57,114 - synapse.http.server - 112 - ERROR - PUT-26389- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:58,340 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:58,342 - synapse.federation.transport.server - 359 - ERROR - PUT-26390- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:58,345 - synapse.http.server - 112 - ERROR - PUT-26390- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:59,150 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:59,152 - synapse.federation.transport.server - 359 - ERROR - PUT-26391- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:59,156 - synapse.http.server - 112 - ERROR - PUT-26391- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:59,619 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:59,622 - synapse.federation.transport.server - 359 - ERROR - PUT-26392- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:30:59,626 - synapse.http.server - 112 - ERROR - PUT-26392- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:00,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:00,502 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2662- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:00,869 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:00,871 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1351- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:02,482 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:02,486 - synapse.federation.transport.server - 359 - ERROR - PUT-26393- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:02,490 - synapse.http.server - 112 - ERROR - PUT-26393- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:03,259 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:03,263 - synapse.federation.transport.server - 359 - ERROR - PUT-26394- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:03,267 - synapse.http.server - 112 - ERROR - PUT-26394- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:03,736 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:03,740 - synapse.federation.transport.server - 359 - ERROR - PUT-26395- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:03,749 - synapse.http.server - 112 - ERROR - PUT-26395- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:04,168 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:04,172 - synapse.federation.transport.server - 359 - ERROR - PUT-26396- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:04,175 - synapse.http.server - 112 - ERROR - PUT-26396- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:04,401 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:04,404 - synapse.federation.transport.server - 359 - ERROR - PUT-26397- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:04,409 - synapse.http.server - 112 - ERROR - PUT-26397- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:04,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:04,502 - synapse.federation.transport.server - 359 - ERROR - PUT-26398- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:04,504 - synapse.http.server - 112 - ERROR - PUT-26398- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:05,341 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:05,342 - synapse.federation.transport.server - 359 - ERROR - PUT-26399- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:05,346 - synapse.http.server - 112 - ERROR - PUT-26399- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:05,505 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:05,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2663- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:07,047 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:07,052 - synapse.federation.transport.server - 359 - ERROR - PUT-26400- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:07,055 - synapse.http.server - 112 - ERROR - PUT-26400- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:08,662 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:08,665 - synapse.federation.transport.server - 359 - ERROR - PUT-26401- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:08,668 - synapse.http.server - 112 - ERROR - PUT-26401- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,215 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:10,218 - synapse.federation.transport.server - 359 - ERROR - PUT-26402- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,223 - synapse.http.server - 112 - ERROR - PUT-26402- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,245 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:10,247 - synapse.federation.transport.server - 359 - ERROR - PUT-26403- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,250 - synapse.http.server - 112 - ERROR - PUT-26403- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,503 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:31:10,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2664- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,522 - synapse.federation.transport.server - 359 - ERROR - PUT-26404- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,526 - synapse.http.server - 112 - ERROR - PUT-26404- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:10,872 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1352- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:12,178 - synapse.federation.transport.server - 359 - ERROR - PUT-26405- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:12,181 - synapse.http.server - 112 - ERROR - PUT-26405- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:13,397 - synapse.federation.transport.server - 359 - ERROR - PUT-26406- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:13,401 - synapse.http.server - 112 - ERROR - PUT-26406- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:14,078 - synapse.federation.transport.server - 359 - ERROR - PUT-26407- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:14,082 - synapse.http.server - 112 - ERROR - PUT-26407- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:15,511 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2665- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:17,337 - synapse.federation.transport.server - 359 - ERROR - PUT-26408- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:17,341 - synapse.http.server - 112 - ERROR - PUT-26408- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:18,893 - synapse.federation.transport.server - 359 - ERROR - PUT-26409- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:18,897 - synapse.http.server - 112 - ERROR - PUT-26409- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:19,699 - synapse.http.server - 112 - ERROR - PUT-26410- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:19,928 - synapse.federation.transport.server - 359 - ERROR - PUT-26411- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:19,930 - synapse.http.server - 112 - ERROR - PUT-26411- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:20,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2666- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:20,822 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-225- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:20,862 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-225- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:20,874 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1353- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:21,081 - synapse.federation.transport.server - 359 - ERROR - PUT-26412- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:21,083 - synapse.http.server - 112 - ERROR - PUT-26412- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:21,528 - synapse.federation.transport.server - 359 - ERROR - PUT-26413- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:21,532 - synapse.http.server - 112 - ERROR - PUT-26413- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:21,859 - synapse.federation.transport.server - 359 - ERROR - PUT-26414- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:21,862 - synapse.http.server - 112 - ERROR - PUT-26414- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:24,336 - synapse.federation.transport.server - 359 - ERROR - PUT-26415- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:24,339 - synapse.http.server - 112 - ERROR - PUT-26415- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:24,703 - synapse.federation.transport.server - 359 - ERROR - PUT-26416- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:24,707 - synapse.http.server - 112 - ERROR - PUT-26416- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:25,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2667- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:25,536 - synapse.federation.transport.server - 359 - ERROR - PUT-26417- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:25,538 - synapse.http.server - 112 - ERROR - PUT-26417- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:26,083 - synapse.http.server - 112 - ERROR - PUT-26420- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:26,089 - synapse.federation.transport.server - 359 - ERROR - PUT-26419- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:26,095 - synapse.http.server - 112 - ERROR - PUT-26419- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:26,491 - synapse.federation.transport.server - 359 - ERROR - PUT-26422- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:26,495 - synapse.http.server - 112 - ERROR - PUT-26422- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:27,169 - synapse.federation.transport.server - 359 - ERROR - PUT-26423- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:27,176 - synapse.http.server - 112 - ERROR - PUT-26423- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:27,627 - synapse.federation.transport.server - 359 - ERROR - PUT-26424- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:27,631 - synapse.http.server - 112 - ERROR - PUT-26424- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:28,863 - synapse.federation.transport.server - 359 - ERROR - PUT-26426- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:28,866 - synapse.http.server - 112 - ERROR - PUT-26426- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:29,271 - synapse.federation.transport.server - 359 - ERROR - PUT-26428- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:29,275 - synapse.http.server - 112 - ERROR - PUT-26428- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:29,327 - synapse.http.server - 112 - ERROR - PUT-26429- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:30,146 - synapse.federation.transport.server - 359 - ERROR - PUT-26431- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:30,152 - synapse.http.server - 112 - ERROR - PUT-26431- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:30,515 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2668- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:30,877 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1354- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:31,488 - synapse.http.server - 112 - ERROR - PUT-26432- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:32,207 - synapse.federation.transport.server - 359 - ERROR - PUT-26433- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:32,212 - synapse.http.server - 112 - ERROR - PUT-26433- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:34,434 - synapse.federation.transport.server - 359 - ERROR - PUT-26434- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:34,441 - synapse.http.server - 112 - ERROR - PUT-26434- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:34,722 - synapse.federation.transport.server - 359 - ERROR - PUT-26435- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:34,728 - synapse.http.server - 112 - ERROR - PUT-26435- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:35,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2669- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:35,638 - synapse.http.server - 112 - ERROR - PUT-26436- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:37,935 - synapse.federation.transport.server - 359 - ERROR - PUT-26437- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:37,938 - synapse.http.server - 112 - ERROR - PUT-26437- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:38,579 - synapse.federation.transport.server - 359 - ERROR - PUT-26438- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:38,581 - synapse.http.server - 112 - ERROR - PUT-26438- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:39,540 - synapse.federation.transport.server - 359 - ERROR - PUT-26439- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:39,544 - synapse.http.server - 112 - ERROR - PUT-26439- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:39,745 - synapse.federation.transport.server - 359 - ERROR - PUT-26440- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:39,749 - synapse.http.server - 112 - ERROR - PUT-26440- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:40,508 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2670- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:40,875 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1355- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:41,743 - synapse.federation.transport.server - 359 - ERROR - PUT-26441- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:41,747 - synapse.http.server - 112 - ERROR - PUT-26441- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:42,300 - synapse.federation.transport.server - 359 - ERROR - PUT-26442- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:42,304 - synapse.http.server - 112 - ERROR - PUT-26442- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:43,820 - synapse.http.server - 112 - ERROR - PUT-26443- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:45,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2671- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:50,375 - synapse.http.server - 112 - ERROR - PUT-26444- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:50,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2672- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:50,881 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1356- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:51,517 - synapse.federation.transport.server - 359 - ERROR - PUT-26445- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:51,519 - synapse.http.server - 112 - ERROR - PUT-26445- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:53,155 - synapse.federation.transport.server - 359 - ERROR - PUT-26446- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:53,158 - synapse.http.server - 112 - ERROR - PUT-26446- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:54,373 - synapse.federation.transport.server - 359 - ERROR - PUT-26448- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:54,376 - synapse.http.server - 112 - ERROR - PUT-26448- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:54,458 - synapse.federation.transport.server - 359 - ERROR - PUT-26447- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:54,460 - synapse.http.server - 112 - ERROR - PUT-26447- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:55,173 - synapse.federation.transport.server - 359 - ERROR - PUT-26449- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:55,181 - synapse.http.server - 112 - ERROR - PUT-26449- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:55,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2673- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:55,584 - synapse.federation.transport.server - 359 - ERROR - PUT-26450- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:55,588 - synapse.http.server - 112 - ERROR - PUT-26450- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:55,864 - synapse.federation.transport.server - 359 - ERROR - PUT-26451- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:55,868 - synapse.http.server - 112 - ERROR - PUT-26451- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:56,748 - synapse.federation.transport.server - 359 - ERROR - PUT-26454- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:56,753 - synapse.http.server - 112 - ERROR - PUT-26454- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:59,190 - synapse.federation.transport.server - 359 - ERROR - PUT-26455- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:31:59,196 - synapse.http.server - 112 - ERROR - PUT-26455- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:00,256 - synapse.http.server - 112 - ERROR - PUT-26456- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:00,629 - synapse.http.server - 112 - ERROR - PUT-26457- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:00,807 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2674- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:00,877 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1357- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:01,112 - synapse.federation.transport.server - 359 - ERROR - PUT-26458- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:01,114 - synapse.http.server - 112 - ERROR - PUT-26458- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:04,045 - synapse.http.server - 112 - ERROR - PUT-26459- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:05,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2675- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:05,751 - synapse.federation.transport.server - 359 - ERROR - PUT-26460- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:05,754 - synapse.http.server - 112 - ERROR - PUT-26460- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:06,418 - synapse.http.server - 112 - ERROR - PUT-26461- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:06,693 - synapse.federation.transport.server - 359 - ERROR - PUT-26462- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:06,697 - synapse.http.server - 112 - ERROR - PUT-26462- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:07,023 - synapse.federation.transport.server - 359 - ERROR - PUT-26463- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:07,027 - synapse.http.server - 112 - ERROR - PUT-26463- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:07,110 - synapse.federation.transport.server - 359 - ERROR - PUT-26464- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:07,114 - synapse.http.server - 112 - ERROR - PUT-26464- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:09,000 - synapse.http.server - 112 - ERROR - PUT-26465- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:10,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2676- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:10,885 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1358- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:15,143 - synapse.federation.transport.server - 359 - ERROR - PUT-26466- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:15,147 - synapse.http.server - 112 - ERROR - PUT-26466- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:15,644 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2677- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:16,110 - synapse.federation.transport.server - 359 - ERROR - PUT-26467- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:16,117 - synapse.http.server - 112 - ERROR - PUT-26467- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:16,127 - synapse.federation.transport.server - 359 - ERROR - PUT-26468- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:16,129 - synapse.http.server - 112 - ERROR - PUT-26468- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:17,373 - synapse.http.server - 112 - ERROR - PUT-26469- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:18,380 - synapse.federation.transport.server - 359 - ERROR - PUT-26470- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:18,384 - synapse.http.server - 112 - ERROR - PUT-26470- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:19,560 - synapse.federation.transport.server - 359 - ERROR - PUT-26471- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:19,564 - synapse.http.server - 112 - ERROR - PUT-26471- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:20,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2678- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:20,830 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-226- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:21,120 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-226- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:21,340 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1359- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:22,888 - synapse.federation.transport.server - 359 - ERROR - PUT-26472- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:22,891 - synapse.http.server - 112 - ERROR - PUT-26472- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:25,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2679- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:30,512 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2680- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:33,073 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1360- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 436, in _expire_url_cache_data media_ids = yield self.store.get_url_cache_media_before(expire_before) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:34,024 - synapse.http.server - 112 - ERROR - PUT-26476- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:34,720 - synapse.federation.federation_server - 809 - ERROR - PUT-26475- Failed to handle edu 'm.presence' Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 805, in on_edu yield handler(origin, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 682, in incoming_presence yield self._update_states(updates) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 332, in _update_states yield self._persist_and_notify(list(to_notify.values())) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/presence.py", line 601, in _persist_and_notify stream_id, max_token = yield self.store.update_presence(states) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/presence.py", line 78, in update_presence self._update_presence_txn, stream_orderings, presence_states, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:35,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2681- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:38,676 - synapse.http.server - 112 - ERROR - PUT-26477- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:40,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:40,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2682- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:40,876 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1361- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:45,089 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:45,093 - synapse.federation.transport.server - 359 - ERROR - PUT-26478- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:45,098 - synapse.http.server - 112 - ERROR - PUT-26478- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:45,498 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:45,500 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2683- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:46,472 - synapse.federation.transport.server - 147 - ERROR - PUT-26479- Error resetting retry timings on matrix.logansnow.com Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 145, in _reset_retry_timings yield self.store.set_destination_retry_timings(origin, 0, 0) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:46,477 - synapse.federation.transport.server - 359 - ERROR - PUT-26479- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:46,480 - synapse.http.server - 112 - ERROR - PUT-26479- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:50,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2684- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:50,870 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:50,874 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1362- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 241, in _new_transaction txn = conn.cursor() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:54,732 - synapse.http.server - 112 - ERROR - PUT-26480- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:55,233 - synapse.http.server - 112 - ERROR - PUT-26481- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:55,318 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:55,323 - synapse.http.server - 112 - ERROR - PUT-26482- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:55,505 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2685- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:57,971 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:57,975 - synapse.http.server - 112 - ERROR - PUT-26486- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:32:58,299 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:32:58,303 - synapse.http.server - 112 - ERROR - PUT-26487- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:00,679 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2686- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:00,873 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1363- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:04,124 - synapse.http.server - 112 - ERROR - PUT-26488- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:05,503 - twisted - 244 - CRITICAL - - Rollback failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) --- --- File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 303, in _runWithConnection conn.rollback() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 52, in rollback self._connection.rollback() psycopg2.InterfaceError: connection already closed 2018-11-24 20:33:05,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2687- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 368, in inner_func conn.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:10,507 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2688- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:10,579 - synapse.http.server - 112 - ERROR - PUT-26489- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:10,874 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1364- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:15,979 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2689- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:17,443 - synapse.http.server - 112 - ERROR - PUT-26490- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:19,065 - synapse.http.server - 112 - ERROR - PUT-26491- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:19,648 - synapse.federation.transport.server - 359 - ERROR - PUT-26492- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:19,652 - synapse.http.server - 112 - ERROR - PUT-26492- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:20,506 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2690- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:20,883 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1365- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:21,056 - synapse.metrics.background_process_metrics - 203 - ERROR - Update remote profile-227- Background process 'Update remote profile' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/profile.py", line 305, in _update_remote_profile_cache last_checked=self.clock.time_msec() - self.PROFILE_UPDATE_EVERY_MS File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:21,219 - synapse.metrics.background_process_metrics - 203 - ERROR - update_recently_accessed_media-227- Background process 'update_recently_accessed_media' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 122, in _update_recently_accessed local_media, remote_media, self.clock.time_msec() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:21,267 - synapse.http.server - 112 - ERROR - PUT-26493- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:23,765 - synapse.http.server - 112 - ERROR - GET-26494- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v2_alpha/groups.py", line 47, in on_GET requester_user_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/groups_local.py", line 50, in h failure.trap(HttpResponseException) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 439, in trap self.raiseException() File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 467, in raiseException raise self.value.with_traceback(self.tb) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/matrixfederationclient.py", line 616, in get_json ignore_backoff=ignore_backoff, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/matrixfederationclient.py", line 261, in _send_request ignore_backoff=ignore_backoff, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/retryutils.py", line 79, in get_retry_limiter destination File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:25,523 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2691- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:25,658 - synapse.http.server - 112 - ERROR - PUT-26495- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:25,736 - synapse.http.server - 112 - ERROR - PUT-26496- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:30,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2692- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:30,873 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1366- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:32,448 - synapse.http.server - 112 - ERROR - PUT-26499- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:32,929 - synapse.http.server - 112 - ERROR - PUT-26500- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:34,047 - synapse.federation.transport.server - 359 - ERROR - PUT-26501- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:34,050 - synapse.http.server - 112 - ERROR - PUT-26501- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:35,503 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2693- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:36,597 - synapse.http.server - 112 - ERROR - PUT-26503- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:37,111 - synapse.http.server - 112 - ERROR - PUT-26505- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:37,996 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:33:38,840 - synapse.http.server - 112 - ERROR - PUT-26506- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:39,971 - synapse.http.server - 112 - ERROR - PUT-26507- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:40,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2694- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:40,885 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1367- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:43,413 - synapse.http.server - 112 - ERROR - PUT-26508- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:45,501 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2695- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:46,148 - synapse.federation.transport.server - 359 - ERROR - PUT-26509- on_incoming_transaction failed Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:46,153 - synapse.http.server - 112 - ERROR - PUT-26509- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 274, in new_func origin, content, request.args, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 356, in on_PUT origin, transaction_data, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 121, in on_incoming_transaction origin, transaction, request_time, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 138, in _handle_incoming_transaction response = yield self.transaction_actions.have_responded(origin, transaction) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: could not translate host name "psql-proxy.service.1.cluster.deuxfleurs.fr" to address: Name or service not known 2018-11-24 20:33:50,504 - synapse.metrics.background_process_metrics - 203 - ERROR - update_client_ips-2696- Background process 'update_client_ips' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:50,878 - synapse.http.server - 112 - ERROR - PUT-26510- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:50,887 - synapse.metrics.background_process_metrics - 203 - ERROR - expire_url_cache_data-1368- Background process 'expire_url_cache_data' threw an exception Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/metrics/background_process_metrics.py", line 201, in run yield func(*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 404, in _expire_url_cache_data media_ids = yield self.store.get_expired_url_cache(now) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:51,989 - synapse.http.server - 112 - ERROR - PUT-26511- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/client/v1/room.py", line 217, in on_POST txn_id=txn_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 488, in create_and_send_nonmember_event txn_id=txn_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 318, in create_event prev_events_and_hashes=prev_events_and_hashes, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/util/metrics.py", line 63, in measured_func r = yield func(self, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/message.py", line 536, in create_new_client_event yield self.store.get_prev_events_for_room(builder.room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/event_federation.py", line 154, in get_prev_events_for_room res = yield self.get_latest_event_ids_and_hashes_in_room(room_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:52,485 - synapse.http.server - 112 - ERROR - PUT-26512- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:33:53,367 - synapse.http.server - 112 - ERROR - PUT-26513- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 316, in _async_render callback_return = yield callback(request, **kwargs) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 260, in new_func origin = yield authenticator.authenticate_request(request, content) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/federation/transport/server.py", line 135, in authenticate_request retry_timings = yield self.store.get_destination_retry_timings(origin) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/transactions.py", line 185, in get_destination_retry_timings self._get_destination_retry_timings, destination) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 295, in _runWithConnection conn = self.connectionFactory(self) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 36, in __init__ self.reconnect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 76, in reconnect self._connection = self._pool.connect() File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 429, in connect conn = self.dbapi.connect(*self.connargs, **self.connkw) File "/usr/local/lib/python3.5/dist-packages/psycopg2/__init__.py", line 130, in connect conn = _connect(dsn, connection_factory=connection_factory, **kwasync) psycopg2.OperationalError: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. 2018-11-24 20:43:10,905 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:44:24,411 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:54:49,034 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:54:50,401 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:54:57,392 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:55:03,545 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:55:21,977 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:55:27,522 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 20:59:55,385 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:01:26,776 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:01:28,502 - synapse.rest.media.v1.preview_url_resource - 363 - ERROR - GET-30134- Error handling downloaded https://tn.smilevideo.jp/smile?i=7692428: TypeError("Can't convert 'bytes' object to str implicitly",) 2018-11-24 21:01:28,505 - synapse.http.server - 112 - ERROR - GET-30134- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 162, in _async_render_GET og = yield make_deferred_yieldable(observable.observe()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 244, in _do_preview _rebase_url(og['og:image'], media_info['uri']), user File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 328, in _download_url _, params = cgi.parse_header(content_disposition[0],) File "/usr/lib/python3.5/cgi.py", line 319, in parse_header parts = _parseparam(';' + line) builtins.TypeError: Can't convert 'bytes' object to str implicitly 2018-11-24 21:02:38,963 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:05:46,872 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:06:46,777 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:14:53,989 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:26:44,163 - synapse.http.server - 112 - ERROR - GET-32518- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 501, in errback self._startRunCallbacks(fail) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks self._runCallbacks() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 162, in _async_render_GET og = yield make_deferred_yieldable(observable.observe()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 244, in _do_preview _rebase_url(og['og:image'], media_info['uri']), user File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 328, in _download_url _, params = cgi.parse_header(content_disposition[0],) File "/usr/lib/python3.5/cgi.py", line 319, in parse_header parts = _parseparam(';' + line) builtins.TypeError: Can't convert 'bytes' object to str implicitly 2018-11-24 21:27:01,102 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:27:58,210 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:28:23,130 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:29:04,716 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:31:30,857 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:35:03,710 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:36:08,173 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:37:10,257 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:40:17,268 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:40:20,684 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:54:09,662 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:54:21,433 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:54:42,938 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:57:21,662 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 21:58:33,854 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:00:02,681 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:01:23,572 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:04:47,860 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:15:26,942 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:15:36,313 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:17:00,856 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:17:05,401 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:18:34,488 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:19:32,852 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:19:34,968 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:20:28,669 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:20:49,464 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:21:59,472 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:23:53,470 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:24:52,409 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:25:19,230 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:26:47,290 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:38:22,136 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:38:25,661 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:38:43,384 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:39:22,552 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:40:11,897 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:42:12,473 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:47:06,936 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:48:33,976 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:49:35,673 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 22:50:15,795 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:18:21,305 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:40:22,265 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:50:27,449 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:50:32,376 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:50:48,248 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:53:39,002 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:57:14,554 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:57:49,775 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:57:57,667 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:58:19,476 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:58:21,053 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:58:28,219 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:58:32,417 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:59:20,397 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-24 23:59:53,769 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:00:32,456 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:00:52,211 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:01:47,838 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:02:26,617 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:03:11,204 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:03:14,048 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:04:36,761 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:05:06,040 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:06:00,392 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:07:11,462 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:08:16,733 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:12:14,486 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:12:19,513 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:12:21,274 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:12:34,617 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:12:42,936 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:13:06,365 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:13:37,892 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:14:53,636 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:15:22,044 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:15:58,460 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:17:01,584 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:18:04,400 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:18:39,417 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:19:30,821 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:20:18,233 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:20:39,579 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:21:27,608 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:21:48,796 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:23:14,616 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:24:30,910 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:35:37,063 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:35:49,942 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:36:00,756 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:36:15,394 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:37:31,192 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:37:33,282 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:38:37,285 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:39:39,837 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:40:14,521 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:40:43,613 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:41:19,545 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:41:57,264 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:42:31,225 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:43:02,261 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:44:25,084 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:45:28,587 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:46:14,456 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:47:25,881 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:47:55,946 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:48:02,360 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:48:25,329 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:49:31,607 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:50:22,046 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:50:33,728 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:51:28,244 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:51:37,067 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:51:49,497 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:52:46,634 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:54:09,669 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:55:05,853 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:55:11,729 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:56:06,081 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:56:31,613 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:57:18,668 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 00:57:55,320 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:09:38,841 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:09:45,778 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:10:09,517 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:11:08,443 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:12:06,001 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:13:08,225 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:14:11,555 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:15:33,017 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:16:35,495 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:17:50,748 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:18:49,457 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:42:29,560 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:49:48,601 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:53:04,342 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:53:21,081 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:54:04,825 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:55:06,212 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:56:22,019 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:57:34,276 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:58:42,997 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:59:45,337 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 01:59:53,423 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:00:07,609 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:00:42,684 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:03:03,511 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:03:18,705 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:04:23,606 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:05:43,737 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:07:42,396 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:08:34,745 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:09:39,382 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:10:34,590 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:10:41,395 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:10:46,998 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:11:09,796 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:12:04,722 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:13:01,549 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:14:11,525 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:15:30,346 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:16:51,382 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:17:52,384 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:18:09,857 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:19:11,051 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:20:04,905 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:20:58,462 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:28:05,598 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:29:11,993 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:30:07,033 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:30:29,200 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:31:31,202 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:32:09,915 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:33:20,569 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:33:38,927 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:34:43,512 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:34:49,055 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:35:59,358 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:36:03,128 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:36:56,175 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:39:36,377 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:40:54,712 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:42:14,587 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:45:45,664 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:46:04,693 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:47:02,353 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:47:19,956 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:49:22,067 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 02:53:05,709 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:09:47,673 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:10:10,609 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:11:10,362 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:14:26,629 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:15:43,656 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:16:51,155 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:18:14,768 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:19:14,789 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:22:35,861 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:44:50,105 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:49:26,328 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:54:43,000 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 03:56:16,953 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:15:04,753 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:16:52,065 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:18:14,986 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:20:41,037 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:21:29,536 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:22:56,701 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:45:01,005 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:45:34,578 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:46:09,534 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:46:31,447 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:47:01,236 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:48:25,208 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:53:58,259 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:55:24,794 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 04:57:08,468 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:10:26,955 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:10:41,423 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:18:51,769 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:20:44,916 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:21:51,993 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:23:30,548 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:25:06,296 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:27:54,740 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:31:03,925 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:49:35,991 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:50:24,375 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:51:52,953 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:54:09,911 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 05:58:06,199 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:02:14,263 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:17:15,977 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:18:52,920 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:19:18,126 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:21:44,440 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:26:44,733 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:28:16,371 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:29:26,525 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:30:37,437 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:38:35,022 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:38:43,539 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:39:06,940 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:40:11,447 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:41:12,783 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:42:32,720 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:43:30,328 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:44:40,826 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:45:38,843 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:46:46,437 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 06:47:47,188 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 07:28:51,730 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 07:32:03,411 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:15:57,625 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:16:18,810 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:16:42,425 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:17:36,343 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:17:52,799 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:18:41,721 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:23:41,556 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:28:32,057 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:29:33,753 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:30:54,137 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:36:25,206 - synapse.rest.media.v1.preview_url_resource - 363 - ERROR - GET-84294- Error handling downloaded https://tn.smilevideo.jp/smile?i=7692428: TypeError("Can't convert 'bytes' object to str implicitly",) 2018-11-25 08:36:25,208 - synapse.http.server - 112 - ERROR - GET-84294- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 162, in _async_render_GET og = yield make_deferred_yieldable(observable.observe()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 244, in _do_preview _rebase_url(og['og:image'], media_info['uri']), user File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 328, in _download_url _, params = cgi.parse_header(content_disposition[0],) File "/usr/lib/python3.5/cgi.py", line 319, in parse_header parts = _parseparam(';' + line) builtins.TypeError: Can't convert 'bytes' object to str implicitly 2018-11-25 08:41:00,877 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:43:06,964 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:44:15,803 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:45:27,188 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:48:41,644 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:54:15,310 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 08:55:08,187 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:00:29,982 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:15:23,869 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:18:08,664 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:19:26,612 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:22:26,312 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:50:28,281 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:52:33,209 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:53:37,721 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:55:06,554 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 09:59:36,642 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:02:20,917 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:16:48,776 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:16:53,618 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:17:21,326 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:17:32,395 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:17:37,500 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:17:54,350 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:18:24,473 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:18:29,193 - synapse.http.server - 112 - ERROR - GET-92131- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 236, in _respond_remote_thumbnail media_info = yield self.media_repo.get_remote_media_info(server_name, media_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 282, in get_remote_media_info server_name, media_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 332, in _get_remote_media_impl server_name, media_id, file_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 442, in _download_remote_file filesystem_id=file_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 246, in _new_transaction r = func(txn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 453, in _simple_insert_txn txn.execute(sql, vals) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 87, in execute self._do_execute(self.txn.execute, sql, *args) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 117, in _do_execute sql, *args psycopg2.IntegrityError: duplicate key value violates unique constraint "remote_media_cache_media_origin_media_id_key" DETAIL: Key (media_origin, media_id)=(matrix.org, xMgOOsVmZukbhtoUrhpimCoO) already exists. 2018-11-25 10:18:48,632 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:19:37,683 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:19:41,316 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:19:47,512 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:20:00,863 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:20:34,910 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:21:32,935 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:21:51,101 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:22:23,890 - synapse.rest.media.v1.media_repository - 394 - ERROR - GET-92155- Failed to fetch remote media dev.t2bot.io/UxZCrhPJJlBbbIuwPTwTmIcH Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 370, in _download_remote_file "allow_remote": "false", twisted.web._newclient.ResponseNeverReceived: [] 2018-11-25 10:22:55,884 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:23:08,084 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:23:19,709 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:23:20,508 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:24:12,767 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:24:36,024 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:25:13,248 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:26:15,343 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:26:21,396 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:26:23,272 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:33:15,064 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:33:28,505 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:36:39,047 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:37:54,025 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:38:57,784 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:41:29,478 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:42:42,603 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:43:03,205 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:43:09,112 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:44:17,976 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:44:40,532 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:51:16,793 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:53:12,403 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:53:13,146 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:53:16,691 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:54:21,628 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:54:42,305 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:55:38,235 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:55:42,752 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:55:46,467 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:55:47,992 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:56:07,714 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:57:18,063 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:58:31,441 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:59:10,616 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 10:59:23,173 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:00:31,320 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:01:04,618 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:11:05,913 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:18:44,875 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:21:55,660 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:23:21,972 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:24:09,529 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:24:41,258 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:25:31,194 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:28:18,000 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:32:03,197 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:32:19,375 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:33:12,764 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:36:10,239 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:36:14,931 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:37:06,232 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:42:35,197 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:43:46,616 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:44:53,689 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:46:41,133 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 11:47:52,121 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:01:12,903 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:01:46,356 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:03:09,422 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:03:58,547 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:11:54,937 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:12:19,769 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:12:52,792 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:14:29,048 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:14:33,230 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:17:10,721 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:19:40,016 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:20:21,817 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:21:54,232 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:24:00,185 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:25:12,121 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:26:15,441 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:26:44,539 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:33:50,224 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:33:55,830 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:33:56,561 - synapse.http.server - 112 - ERROR - GET-106853- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:33:57,640 - synapse.http.server - 112 - ERROR - GET-106875- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:33:57,658 - synapse.http.server - 112 - ERROR - GET-106876- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 253, in get_remote_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 72, in _async_render_GET yield self.media_repo.get_remote_media(request, server_name, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 253, in get_remote_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:33:57,678 - synapse.http.server - 112 - ERROR - GET-106877- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:33:58,013 - synapse.http.server - 112 - ERROR - GET-106882- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:34:12,977 - synapse.http.server - 112 - ERROR - GET-106922- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:34:22,474 - synapse.http.server - 112 - ERROR - GET-106942- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 236, in _respond_remote_thumbnail media_info = yield self.media_repo.get_remote_media_info(server_name, media_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 282, in get_remote_media_info server_name, media_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 332, in _get_remote_media_impl server_name, media_id, file_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 442, in _download_remote_file filesystem_id=file_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 246, in _new_transaction r = func(txn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 453, in _simple_insert_txn txn.execute(sql, vals) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 87, in execute self._do_execute(self.txn.execute, sql, *args) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 117, in _do_execute sql, *args psycopg2.IntegrityError: duplicate key value violates unique constraint "remote_media_cache_media_origin_media_id_key" DETAIL: Key (media_origin, media_id)=(matrix.org, hfkZpbZNNrHRgLVdjvqVPaeW) already exists. 2018-11-25 12:34:22,713 - synapse.http.server - 112 - ERROR - GET-106946- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 236, in _respond_remote_thumbnail media_info = yield self.media_repo.get_remote_media_info(server_name, media_id) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 282, in get_remote_media_info server_name, media_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 332, in _get_remote_media_impl server_name, media_id, file_id File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 442, in _download_remote_file filesystem_id=file_id, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 433, in _simple_insert self._simple_insert_txn, table, values, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 325, in runInteraction *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 374, in runWithConnection inner_func, *args, **kwargs File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/usr/local/lib/python3.5/dist-packages/twisted/python/threadpool.py", line 266, in inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection compat.reraise(excValue, excTraceback) File "/usr/local/lib/python3.5/dist-packages/twisted/python/compat.py", line 464, in reraise raise exception.with_traceback(traceback) File "/usr/local/lib/python3.5/dist-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection result = func(conn, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 370, in inner_func return func(conn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 246, in _new_transaction r = func(txn, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 453, in _simple_insert_txn txn.execute(sql, vals) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 87, in execute self._do_execute(self.txn.execute, sql, *args) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 117, in _do_execute sql, *args psycopg2.IntegrityError: duplicate key value violates unique constraint "remote_media_cache_media_origin_media_id_key" DETAIL: Key (media_origin, media_id)=(matrix.org, VmvudJEGfxnSOIjjCcjtpvKN) already exists. 2018-11-25 12:34:51,097 - synapse.http.server - 112 - ERROR - GET-107018- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:34:51,116 - synapse.http.server - 112 - ERROR - GET-107017- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:34:51,144 - synapse.http.server - 112 - ERROR - GET-107020- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:35:32,976 - synapse.http.server - 112 - ERROR - GET-107193- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:37:20,587 - synapse.http.server - 112 - ERROR - GET-107524- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 253, in get_remote_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 72, in _async_render_GET yield self.media_repo.get_remote_media(request, server_name, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 253, in get_remote_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:42:40,601 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:46:15,119 - synapse.http.server - 112 - ERROR - GET-108633- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 253, in get_remote_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 72, in _async_render_GET yield self.media_repo.get_remote_media(request, server_name, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 253, in get_remote_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:46:15,128 - synapse.http.server - 112 - ERROR - GET-108634- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:46:15,138 - synapse.http.server - 112 - ERROR - GET-108635- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 12:49:17,904 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:49:48,355 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:50:49,255 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:51:56,945 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 12:54:06,103 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 13:20:15,296 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 13:20:19,997 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 13:43:38,171 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:11:24,223 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:12:35,641 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:13:56,796 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:15:04,893 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:16:23,225 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:17:46,169 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:18:36,384 - synapse.http.server - 112 - ERROR - GET-118814- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:36,405 - synapse.http.server - 112 - ERROR - GET-118816- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:36,421 - synapse.http.server - 112 - ERROR - GET-118817- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:36,927 - synapse.http.server - 112 - ERROR - GET-118827- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:36,951 - synapse.http.server - 112 - ERROR - GET-118824- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:36,975 - synapse.http.server - 112 - ERROR - GET-118828- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:37,012 - synapse.http.server - 112 - ERROR - GET-118833- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:37,016 - synapse.http.server - 112 - ERROR - GET-118835- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:37,273 - synapse.http.server - 112 - ERROR - GET-118842- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:37,298 - synapse.http.server - 112 - ERROR - GET-118843- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:38,359 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:18:38,922 - synapse.http.server - 112 - ERROR - GET-118878- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:18:42,944 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:18:55,679 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:18:57,766 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:19:13,595 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:19:24,219 - synapse.http.server - 112 - ERROR - GET-118980- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,377 - synapse.http.server - 112 - ERROR - GET-118984- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,389 - synapse.http.server - 112 - ERROR - GET-118986- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,410 - synapse.http.server - 112 - ERROR - GET-118985- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,419 - synapse.http.server - 112 - ERROR - GET-118987- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,424 - synapse.http.server - 112 - ERROR - GET-118988- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,466 - synapse.http.server - 112 - ERROR - GET-118991- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,527 - synapse.http.server - 112 - ERROR - GET-118994- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,542 - synapse.http.server - 112 - ERROR - GET-118997- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,555 - synapse.http.server - 112 - ERROR - GET-118998- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,854 - synapse.http.server - 112 - ERROR - GET-119007- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:24,935 - synapse.http.server - 112 - ERROR - GET-119009- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:25,234 - synapse.http.server - 112 - ERROR - GET-119017- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:19:32,666 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:19:46,394 - synapse.http.server - 112 - ERROR - GET-119084- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:20:07,068 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:20:51,769 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:21:10,546 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:21:59,866 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:22:18,642 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:23:19,606 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:24:16,499 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:25:31,120 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:26:44,459 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:27:54,492 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:32:11,142 - synapse.http.server - 112 - ERROR - GET-120895- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,177 - synapse.http.server - 112 - ERROR - GET-120896- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,241 - synapse.http.server - 112 - ERROR - GET-120901- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,267 - synapse.http.server - 112 - ERROR - GET-120902- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,436 - synapse.http.server - 112 - ERROR - GET-120905- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,752 - synapse.http.server - 112 - ERROR - GET-120915- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,761 - synapse.http.server - 112 - ERROR - GET-120917- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,769 - synapse.http.server - 112 - ERROR - GET-120916- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,786 - synapse.http.server - 112 - ERROR - GET-120918- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:32:11,928 - synapse.http.server - 112 - ERROR - GET-120921- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:37:24,721 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:38:02,936 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:39:25,284 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:39:29,526 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:39:44,578 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:40:33,530 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:41:15,192 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:41:27,340 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:42:30,075 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:42:30,969 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:43:35,754 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:43:42,904 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:44:25,371 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:44:51,257 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:45:38,412 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:46:30,110 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:46:36,634 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:47:59,418 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:48:00,672 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:52:22,455 - synapse.http.server - 112 - ERROR - GET-123590- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,465 - synapse.http.server - 112 - ERROR - GET-123593- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,473 - synapse.http.server - 112 - ERROR - GET-123592- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,483 - synapse.http.server - 112 - ERROR - GET-123594- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,497 - synapse.http.server - 112 - ERROR - GET-123598- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,517 - synapse.http.server - 112 - ERROR - GET-123601- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,526 - synapse.http.server - 112 - ERROR - GET-123603- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,542 - synapse.http.server - 112 - ERROR - GET-123604- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,549 - synapse.http.server - 112 - ERROR - GET-123602- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,559 - synapse.http.server - 112 - ERROR - GET-123606- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,567 - synapse.http.server - 112 - ERROR - GET-123610- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,585 - synapse.http.server - 112 - ERROR - GET-123605- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:22,890 - synapse.http.server - 112 - ERROR - GET-123615- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 14:52:24,008 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 14:52:28,161 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:01:45,786 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:02:43,385 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:05:46,168 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:25:44,903 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:26:18,820 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:27:10,826 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:28:19,598 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:29:19,928 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:30:11,416 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:31:17,707 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:32:19,413 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:34:31,942 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:37:44,107 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:37:57,640 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:39:39,477 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:40:54,282 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:43:30,697 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:46:24,009 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:46:43,888 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:46:44,243 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:46:47,852 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:46:49,746 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:47:15,167 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:47:15,984 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:47:16,213 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:48:14,748 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:48:21,503 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:48:39,534 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:48:58,118 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:49:25,216 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:50:14,606 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:51:26,243 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:52:28,831 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:52:36,736 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:53:37,595 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:53:43,168 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:54:35,309 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:55:59,394 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:57:08,115 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 15:59:10,342 - synapse.http.server - 112 - ERROR - GET-133969- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 15:59:10,693 - synapse.http.server - 112 - ERROR - GET-133973- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 15:59:10,715 - synapse.http.server - 112 - ERROR - GET-133974- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 16:02:59,848 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:03:45,929 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:04:47,146 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-25 16:04:47,149 - twisted - 244 - CRITICAL - - twisted.web._newclient.ResponseNeverReceived: [] 2018-11-25 16:05:01,702 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:05:46,509 - synapse.util.async_helpers - 421 - ERROR - - Canceller failed during timeout Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/async_helpers.py", line 419, in time_it_out deferred.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 548, in cancel self.result.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 537, in cancel canceller(self) File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 1481, in cancelRequest self.transport.abortConnection() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 435, in abortConnection self._shutdownTLS() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 338, in _shutdownTLS shutdownSuccess = self._tlsConnection.shutdown() AttributeError: 'NoneType' object has no attribute 'shutdown' 2018-11-25 16:06:09,801 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:11:25,101 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:11:36,679 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:12:30,216 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:13:56,793 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:15:07,912 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:15:54,250 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:16:16,520 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:16:58,296 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:17:57,433 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:18:23,383 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:19:05,720 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:19:25,245 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:19:55,129 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:21:20,377 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:23:05,912 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:25:32,276 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:27:10,333 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:41:19,830 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:42:13,050 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:42:23,178 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:43:24,249 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:43:37,678 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:44:01,798 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:44:20,207 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:45:03,288 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:45:29,841 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:47:20,076 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:48:30,037 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:49:52,465 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:49:54,483 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:52:05,345 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:52:32,085 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:52:41,528 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:53:56,303 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:54:54,973 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:55:15,385 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:57:12,039 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:58:13,733 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:58:17,749 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:58:38,176 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 16:59:57,452 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:00:09,034 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:01:02,850 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:04:40,070 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:08:20,913 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:08:24,742 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:08:41,939 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:10:04,545 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:11:22,723 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:12:36,832 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:13:46,193 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:14:34,587 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:17:07,219 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:17:11,829 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:17:33,889 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:17:45,273 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:18:34,245 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:19:28,760 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:20:31,889 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:21:37,209 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:21:55,085 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:24:40,872 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:31:41,112 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:33:07,897 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:52:17,770 - synapse.http.server - 112 - ERROR - GET-147043- Failed handle request via : : Traceback (most recent call last): --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/download_resource.py", line 60, in _async_render_GET yield self.media_repo.get_local_media(request, media_id, name) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 212, in get_local_media request, responder, media_type, media_length, upload_name, File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) builtins.Exception: Consumer asked us to stop producing 2018-11-25 17:53:58,891 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 17:55:06,731 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:06:55,083 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:18:20,292 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:21:49,356 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:22:55,151 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:23:58,011 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:26:40,669 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:32:32,185 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:33:27,737 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:37:10,712 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:38:40,313 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 18:40:35,768 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 19:22:34,227 - synapse.http.server - 112 - ERROR - GET-159597- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 21:43:15,594 - synapse.http.server - 112 - ERROR - GET-176464- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 21:43:15,623 - synapse.http.server - 112 - ERROR - GET-176465- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 21:43:15,671 - synapse.http.server - 112 - ERROR - GET-176467- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 21:43:16,022 - synapse.http.server - 112 - ERROR - GET-176472- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 21:43:16,268 - synapse.http.server - 112 - ERROR - GET-176484- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 71, in _async_render_GET request, media_id, width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 121, in _respond_local_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-25 21:53:21,785 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 21:55:45,910 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 21:55:49,958 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:05:05,314 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:05:08,946 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:23:53,509 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:23:58,872 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:24:17,423 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:26:39,698 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:36:35,051 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:41:18,958 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:43:33,641 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:43:38,839 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:44:49,505 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:48:08,167 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:48:51,050 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:50:20,089 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 22:52:48,494 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:16:18,964 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:16:20,682 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:16:23,442 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:16:25,659 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:16:37,401 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:17:19,172 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:17:39,482 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:18:07,095 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:18:29,545 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:19:24,557 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:19:36,587 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:20:39,029 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:21:20,327 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:21:38,584 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:22:52,055 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:22:56,569 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:23:12,068 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:24:02,105 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:24:04,724 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:25:04,465 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:25:23,771 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:45:54,617 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:47:18,584 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:47:58,456 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:48:57,865 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-25 23:50:49,785 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:09:41,585 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:13:07,901 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:14:35,483 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:15:55,599 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:17:31,576 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:18:19,689 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-26 00:18:19,691 - twisted - 244 - CRITICAL - - twisted.web._newclient.ResponseNeverReceived: [] 2018-11-26 00:19:09,696 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:19:15,122 - synapse.util.async_helpers - 421 - ERROR - - Canceller failed during timeout Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/async_helpers.py", line 419, in time_it_out deferred.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 548, in cancel self.result.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 537, in cancel canceller(self) File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 1481, in cancelRequest self.transport.abortConnection() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 435, in abortConnection self._shutdownTLS() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 338, in _shutdownTLS shutdownSuccess = self._tlsConnection.shutdown() AttributeError: 'NoneType' object has no attribute 'shutdown' 2018-11-26 00:24:12,729 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:24:28,600 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:25:11,353 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:26:18,942 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:27:34,993 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:28:58,424 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 00:38:54,141 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:01:56,025 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:05:10,841 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:06:17,913 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:43:57,625 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:44:29,372 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:45:47,452 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:51:01,816 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 01:52:30,649 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:04:05,210 - synapse.state.v1 - 196 - ERROR - PUT-212880-$1543197833111krMht:encom.eu.org-$2458-4-4-6T06252.913452|Marina's_Silly_Donkey_Sits_Snowpain's_Good_Newt:ponies.im- Failed to resolve state Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 364, in _enqueue_events for e in res if e File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: {"$2458-4-4-1T34409.752380|Queen Chrysalis's\u205fUpset\u205fKangaroo\u205fSteals\u205fCranky Doodle Ryan\u2019s\u205fFantastic\u205fLeopard:ponies.im": _EventCacheEntry(event=, redacted_event=None)} During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 197, in _get_events defer.returnValue(events) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: [] During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 116, in get_event defer.returnValue(event) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 02:04:05,221 - synapse.federation.federation_server - 227 - ERROR - PUT-212880-$1543197833111krMht:encom.eu.org- Failed to handle PDU $1543197833111krMht:encom.eu.org: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 608, in resolve_events_with_store state_sets, event_map, state_res_store.get_events, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 216, in process_pdus_for_room origin, pdu File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 640, in _handle_received_pdu origin, pdu, sent_to_us_directly=True, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 262, in on_receive_pdu origin, pdu, prevs, min_depth File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 531, in _get_missing_events_for_pdu sent_to_us_directly=False, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 421, in on_receive_pdu auth_chain=auth_chain, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 594, in _process_received_pdu state=state, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1564, in _handle_new_event auth_events=auth_events, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1762, in _prep_event origin, event, context, auth_events=auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1963, in do_auth event File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 108, in resolve_events_with_store unconflicted_state, conflicted_state, auth_events, state_map File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() builtins.UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 02:25:35,889 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:25:55,329 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:26:18,877 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:28:05,629 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:29:59,805 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:34:00,571 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:34:05,563 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:34:17,588 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:34:32,441 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:34:34,229 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:34:37,560 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:35:15,445 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:36:06,393 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:36:07,417 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:37:26,014 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:37:37,534 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:38:22,337 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:38:52,030 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:39:34,261 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:39:56,539 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:41:25,631 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:41:52,512 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:42:41,918 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:43:17,559 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:43:23,641 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:44:15,165 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:44:46,870 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:44:53,760 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:46:26,688 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:47:53,219 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:47:54,751 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:49:13,341 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:52:46,146 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:53:01,241 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:58:44,472 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 02:59:32,345 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:00:59,128 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:03:08,404 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:06:24,931 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:06:25,367 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:06:50,428 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:06:59,698 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:07:47,705 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:08:51,710 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:10:07,736 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:13:08,537 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:14:30,201 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:16:09,529 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:17:22,427 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:17:45,273 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:20:38,778 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:24:01,017 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:25:23,454 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:26:33,588 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:27:29,918 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:34:36,217 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:36:07,097 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:38:38,909 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:39:59,033 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:43:09,986 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:44:17,593 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:49:29,661 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 03:54:22,009 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:05:38,305 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:06:18,746 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:07:48,664 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:11:30,873 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:14:26,168 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:15:31,773 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:16:05,369 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:17:06,493 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:17:35,348 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:18:34,552 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:18:37,817 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:19:55,193 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:21:36,314 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:23:39,646 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:27:01,118 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:29:34,973 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:31:05,332 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:32:27,769 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:36:39,993 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:37:03,800 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:40:08,889 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:41:26,969 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:55:43,801 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:56:09,145 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 04:58:54,264 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:00:12,601 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:04:00,441 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:05:25,433 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:06:42,498 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:07:56,472 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:29:13,145 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:29:44,888 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:33:13,529 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:38:56,370 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:38:57,647 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:39:01,188 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:39:16,298 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:40:44,090 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 05:44:07,872 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:18:15,609 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:19:05,272 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:20:35,898 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:23:50,200 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:25:24,922 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:43:24,472 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:43:56,217 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:45:43,267 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:45:46,942 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:46:05,491 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:47:29,720 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 06:55:17,694 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:00:44,799 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:12:05,612 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:46:39,928 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:47:35,736 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:51:13,593 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:52:45,753 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:53:44,120 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:55:10,908 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:56:54,073 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 07:59:14,106 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:00:04,954 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:00:10,361 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:03:00,850 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:12:00,205 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:12:52,820 - synapse.rest.media.v1.media_repository - 394 - ERROR - GET-248907- Failed to fetch remote media dev.t2bot.io/UxZCrhPJJlBbbIuwPTwTmIcH Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 370, in _download_remote_file "allow_remote": "false", twisted.web._newclient.ResponseNeverReceived: [] 2018-11-26 08:23:43,760 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:26:58,861 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:30:03,995 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:31:23,518 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:32:25,460 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:33:20,430 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:38:11,070 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:39:05,988 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:39:42,368 - synapse.rest.media.v1.media_repository - 394 - ERROR - GET-253624- Failed to fetch remote media dev.t2bot.io/UxZCrhPJJlBbbIuwPTwTmIcH Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 370, in _download_remote_file "allow_remote": "false", twisted.web._newclient.ResponseNeverReceived: [] 2018-11-26 08:42:21,166 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:44:48,632 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:46:19,513 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:47:17,398 - synapse.http.server - 112 - ERROR - GET-257012- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 83, in _async_render_GET width, height, method, m_type File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/thumbnail_resource.py", line 259, in _respond_remote_thumbnail yield respond_with_responder(request, responder, t_type, t_length) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/_base.py", line 146, in respond_with_responder yield responder.write_to_consumer(request) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_storage.py", line 263, in write_to_consumer FileSender().beginFileTransfer(self.open_file, consumer) File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/basic.py", line 923, in beginFileTransfer self.consumer.registerProducer(self, False) File "/usr/local/lib/python3.5/dist-packages/twisted/web/http.py", line 958, in registerProducer self.channel.registerProducer(producer, streaming) builtins.AttributeError: 'NoneType' object has no attribute 'registerProducer' 2018-11-26 08:47:22,237 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:48:31,357 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:49:55,805 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:50:59,577 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:53:12,555 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:53:36,253 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:54:45,776 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:55:19,395 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 08:56:01,647 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:01:10,947 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:02:36,925 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:03:08,670 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:03:59,613 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:04:47,977 - synapse.state.v1 - 196 - ERROR - PUT-260673-$1543223055125NpvOU:encom.eu.org-$2458-4-4-6T06252.913452|Marina's_Silly_Donkey_Sits_Snowpain's_Good_Newt:ponies.im- Failed to resolve state Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 364, in _enqueue_events for e in res if e File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: {"$2458-4-4-1T34409.752380|Queen Chrysalis's\u205fUpset\u205fKangaroo\u205fSteals\u205fCranky Doodle Ryan\u2019s\u205fFantastic\u205fLeopard:ponies.im": _EventCacheEntry(event=, redacted_event=None)} During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 197, in _get_events defer.returnValue(events) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: [] During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 116, in get_event defer.returnValue(event) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 09:04:47,983 - synapse.federation.federation_server - 227 - ERROR - PUT-260673-$1543223055125NpvOU:encom.eu.org- Failed to handle PDU $1543223055125NpvOU:encom.eu.org: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 608, in resolve_events_with_store state_sets, event_map, state_res_store.get_events, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 216, in process_pdus_for_room origin, pdu File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 640, in _handle_received_pdu origin, pdu, sent_to_us_directly=True, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 262, in on_receive_pdu origin, pdu, prevs, min_depth File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 531, in _get_missing_events_for_pdu sent_to_us_directly=False, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 421, in on_receive_pdu auth_chain=auth_chain, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 594, in _process_received_pdu state=state, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1564, in _handle_new_event auth_events=auth_events, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1762, in _prep_event origin, event, context, auth_events=auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1963, in do_auth event File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 108, in resolve_events_with_store unconflicted_state, conflicted_state, auth_events, state_map File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() builtins.UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 09:05:11,886 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:05:38,914 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:06:03,147 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:08:52,724 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:10:07,993 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:14:03,519 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:15:30,816 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:16:04,394 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:17:09,913 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:18:52,630 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:19:04,146 - synapse.state.v1 - 196 - ERROR - PUT-264308-$1543223920123WOHnx:staubsaugertraum.de-$2458-4-4-6T06252.913452|Marina's_Silly_Donkey_Sits_Snowpain's_Good_Newt:ponies.im- Failed to resolve state Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 364, in _enqueue_events for e in res if e File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: {"$2458-4-4-1T34409.752380|Queen Chrysalis's\u205fUpset\u205fKangaroo\u205fSteals\u205fCranky Doodle Ryan\u2019s\u205fFantastic\u205fLeopard:ponies.im": _EventCacheEntry(event=, redacted_event=None)} During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 197, in _get_events defer.returnValue(events) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: [] During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/events_worker.py", line 116, in get_event defer.returnValue(event) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 09:19:04,152 - synapse.federation.federation_server - 227 - ERROR - PUT-264308-$1543223920123WOHnx:staubsaugertraum.de- Failed to handle PDU $1543223920123WOHnx:staubsaugertraum.de: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 608, in resolve_events_with_store state_sets, event_map, state_res_store.get_events, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 216, in process_pdus_for_room origin, pdu File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 640, in _handle_received_pdu origin, pdu, sent_to_us_directly=True, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 262, in on_receive_pdu origin, pdu, prevs, min_depth File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 531, in _get_missing_events_for_pdu sent_to_us_directly=False, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 421, in on_receive_pdu auth_chain=auth_chain, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 594, in _process_received_pdu state=state, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1564, in _handle_new_event auth_events=auth_events, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1762, in _prep_event origin, event, context, auth_events=auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1963, in do_auth event File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 108, in resolve_events_with_store unconflicted_state, conflicted_state, auth_events, state_map File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() builtins.UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 09:20:10,768 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:21:46,209 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:22:39,125 - synapse.state.v1 - 196 - ERROR - PUT-265654-$1543224126124OVAId:staubsaugertraum.de-$2458-4-4-6T06252.913452|Marina's_Silly_Donkey_Sits_Snowpain's_Good_Newt:ponies.im- Failed to resolve state Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 377, in runWithConnection defer.returnValue(result) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: {'$15325334911003RhjUY:sw1v.org': '', '$152873010877837iaEeS:matrix.org': '', '$2458-4-3-7T10970.937836|Bloomberg`s_Real_Penguin_Respects_Colonel Purple Dart`s_Inquisitive_Jaguar:ponies.im': '', "$2458-4-4-1T34409.752380|Queen Chrysalis's\u205fUpset\u205fKangaroo\u205fSteals\u205fCranky Doodle Ryan\u2019s\u205fFantastic\u205fLeopard:ponies.im": None} During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/storage/_base.py", line 335, in runInteraction defer.returnValue(result) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1362, in returnValue raise _DefGen_Return(val) twisted.internet.defer._DefGen_Return: {'$15325334911003RhjUY:sw1v.org': '', '$152873010877837iaEeS:matrix.org': '', '$2458-4-3-7T10970.937836|Bloomberg`s_Real_Penguin_Respects_Colonel Purple Dart`s_Inquisitive_Jaguar:ponies.im': '', "$2458-4-4-1T34409.752380|Queen Chrysalis's\u205fUpset\u205fKangaroo\u205fSteals\u205fCranky Doodle Ryan\u2019s\u205fFantastic\u205fLeopard:ponies.im": None} During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 09:22:39,129 - synapse.federation.federation_server - 227 - ERROR - PUT-265654-$1543224126124OVAId:staubsaugertraum.de- Failed to handle PDU $1543224126124OVAId:staubsaugertraum.de: Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 608, in resolve_events_with_store state_sets, event_map, state_res_store.get_events, File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1613, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 216, in process_pdus_for_room origin, pdu File "/usr/local/lib/python3.5/dist-packages/synapse/federation/federation_server.py", line 640, in _handle_received_pdu origin, pdu, sent_to_us_directly=True, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 262, in on_receive_pdu origin, pdu, prevs, min_depth File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 531, in _get_missing_events_for_pdu sent_to_us_directly=False, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 421, in on_receive_pdu auth_chain=auth_chain, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 594, in _process_received_pdu state=state, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1564, in _handle_new_event auth_events=auth_events, File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1762, in _prep_event origin, event, context, auth_events=auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/handlers/federation.py", line 1963, in do_auth event File "/usr/local/lib/python3.5/dist-packages/synapse/state/__init__.py", line 400, in resolve_events state_res_store=StateResolutionStore(self.store), File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 108, in resolve_events_with_store unconflicted_state, conflicted_state, auth_events, state_map File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 193, in _resolve_with_state conflicted_state, auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 240, in _resolve_state_events auth_events File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 256, in _resolve_auth_events reverse = [i for i in reversed(_ordered_events(events))] File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 303, in _ordered_events return sorted(events, key=key_func) File "/usr/local/lib/python3.5/dist-packages/synapse/state/v1.py", line 301, in key_func return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest() builtins.UnicodeEncodeError: 'ascii' codec can't encode character '\u205f' in position 42: ordinal not in range(128) 2018-11-26 09:23:27,735 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:23:59,477 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:24:17,459 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:25:13,031 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:25:30,087 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:26:25,239 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:26:35,379 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:27:58,589 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:28:12,916 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:29:17,939 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 09:29:41,139 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-26 09:29:41,141 - twisted - 244 - CRITICAL - - twisted.web._newclient.ResponseNeverReceived: [] 2018-11-26 09:29:41,143 - twisted - 244 - CRITICAL - - Unhandled error in Deferred: 2018-11-26 09:29:41,144 - twisted - 244 - CRITICAL - - twisted.web._newclient.ResponseNeverReceived: [] 2018-11-26 09:30:36,176 - synapse.util.async_helpers - 421 - ERROR - - Canceller failed during timeout Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/async_helpers.py", line 419, in time_it_out deferred.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 548, in cancel self.result.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 537, in cancel canceller(self) File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 1481, in cancelRequest self.transport.abortConnection() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 435, in abortConnection self._shutdownTLS() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 338, in _shutdownTLS shutdownSuccess = self._tlsConnection.shutdown() AttributeError: 'NoneType' object has no attribute 'shutdown' 2018-11-26 09:30:36,855 - synapse.util.async_helpers - 421 - ERROR - - Canceller failed during timeout Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/util/async_helpers.py", line 419, in time_it_out deferred.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 548, in cancel self.result.cancel() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 537, in cancel canceller(self) File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 1481, in cancelRequest self.transport.abortConnection() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 435, in abortConnection self._shutdownTLS() File "/usr/local/lib/python3.5/dist-packages/twisted/protocols/tls.py", line 338, in _shutdownTLS shutdownSuccess = self._tlsConnection.shutdown() AttributeError: 'NoneType' object has no attribute 'shutdown' 2018-11-26 09:30:44,210 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 10:09:09,308 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 10:09:14,401 - twisted - 244 - CRITICAL - - Unexpected exception from twisted.web.client.FileBodyProducer.stopProducing Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/web/_newclient.py", line 195, in _callAppFunction function() File "/usr/local/lib/python3.5/dist-packages/twisted/web/client.py", line 1087, in stopProducing self._task.stop() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 497, in stop self._checkFinish() File "/usr/local/lib/python3.5/dist-packages/twisted/internet/task.py", line 507, in _checkFinish raise self._completionState twisted.internet.task.TaskStopped 2018-11-26 10:10:56,909 - synapse.rest.media.v1.preview_url_resource - 363 - ERROR - GET-275676- Error handling downloaded https://upload.wikimedia.org/wikipedia/commons/thumb/f/f9/Off-The-Record_authentication_dialog..png/220px-Off-The-Record_authentication_dialog..png: TypeError("Can't convert 'bytes' object to str implicitly",) 2018-11-26 10:10:56,910 - synapse.http.server - 112 - ERROR - GET-275676- Failed handle request via : : Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1475, in gotResult _inlineCallbacks(r, g, status) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- --- File "/usr/local/lib/python3.5/dist-packages/synapse/http/server.py", line 81, in wrapped_request_handler yield h(self, request) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 162, in _async_render_GET og = yield make_deferred_yieldable(observable.observe()) File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/usr/local/lib/python3.5/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 244, in _do_preview _rebase_url(og['og:image'], media_info['uri']), user File "/usr/local/lib/python3.5/dist-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/preview_url_resource.py", line 328, in _download_url _, params = cgi.parse_header(content_disposition[0],) File "/usr/lib/python3.5/cgi.py", line 319, in parse_header parts = _parseparam(';' + line) builtins.TypeError: Can't convert 'bytes' object to str implicitly 2018-11-26 10:14:51,267 - synapse.rest.media.v1.media_repository - 394 - ERROR - GET-275652- Failed to fetch remote media dev.t2bot.io/UxZCrhPJJlBbbIuwPTwTmIcH Traceback (most recent call last): File "/usr/local/lib/python3.5/dist-packages/synapse/rest/media/v1/media_repository.py", line 370, in _download_remote_file "allow_remote": "false", twisted.web._newclient.ResponseNeverReceived: []