Keep track when we try and fail to process a pulled event

synapse/federation/federation_base.py

@@ -129,6 +129,13 @@ async def _check_sigs_and_hash(
                     "event_id": pdu.event_id,
                 }
             )
+
+            # TODO: Is it okay to assume this is called from backfilling?
+            #
+            # In any case, we definately don't want to keep fetching spam over
+            # and over and failing it.
+            await self._store.record_event_backfill_attempt(pdu.event_id)
+
             # we redact (to save disk space) as well as soft-failing (to stop
             # using the event in prev_events).
             redacted_event = prune_event(pdu)

synapse/handlers/federation_event.py

@@ -862,6 +862,8 @@ async def _process_pulled_event(
             self._sanity_check_event(event)
         except SynapseError as err:
             logger.warning("Event %s failed sanity check: %s", event_id, err)
+            if backfilled:
+                await self._store.record_event_backfill_attempt(event_id)
             return
 
         try:
@@ -897,6 +899,9 @@ async def _process_pulled_event(
                     backfilled=backfilled,
                 )
         except FederationError as e:
+            if backfilled:
+                await self._store.record_event_backfill_attempt(event_id)
+
             if e.code == 403:
                 logger.warning("Pulled event %s failed history check.", event_id)
             else:

synapse/storage/databases/main/event_federation.py

@@ -1292,6 +1292,22 @@ def _get_backfill_events(
 
         return event_id_results
 
+    @trace
+    async def record_event_backfill_attempt(self, event_id: str) -> None:
+        await self.db_pool.simple_upsert(
+            table="event_backfill_attempts",
+            keyvalues={"event_id": event_id},
+            values={
+                "event_id": event_id,
+                # TODO: This needs to increment
+                "num_attempts": 1,
+                "last_attempt_ts": self._clock.time_msec(),
+            },
+            insertion_values={},
+            desc="record_event_backfill_attempt",
+            lock=False,
+        )
+
     async def get_missing_events(
         self,
         room_id: str,

synapse/storage/databases/main/events.py

@@ -2435,17 +2435,27 @@ def _update_backward_extremeties(
             "DELETE FROM event_backward_extremities"
             " WHERE event_id = ? AND room_id = ?"
         )
+        backward_extremities_to_remove = [
+            (ev.event_id, ev.room_id)
+            for ev in events
+            if not ev.internal_metadata.is_outlier()
+            # If we encountered an event with no prev_events, then we might
+            # as well remove it now because it won't ever have anything else
+            # to backfill from.
+            or len(ev.prev_event_ids()) == 0
+        ]
         txn.execute_batch(
             query,
-            [
-                (ev.event_id, ev.room_id)
-                for ev in events
-                if not ev.internal_metadata.is_outlier()
-                # If we encountered an event with no prev_events, then we might
-                # as well remove it now because it won't ever have anything else
-                # to backfill from.
-                or len(ev.prev_event_ids()) == 0
-            ],
+            backward_extremities_to_remove,
+        )
+
+        # Since we no longer need it as a backward extremity, it won't be
+        # backfilled again so we no longer need to store the backfill attempts
+        # around it.
+        query = "DELETE FROM event_backfill_attempts" " WHERE event_id = ?"
+        txn.execute_batch(
+            query,
+            backward_extremities_to_remove,
         )
 
 

synapse/storage/schema/main/delta/72/04event_backfill_access_time.sql

@@ -0,0 +1,27 @@
+/* Copyright 2022 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+-- Add a table that keeps track of when we last tried to backfill an event. This
+-- allows us to be more intelligent when we decide to retry (we don't need to
+-- fail over and over) and we can process that event in the background so we
+-- don't block on it each time.
+CREATE TABLE IF NOT EXISTS event_backfill_attempts(
+    event_id TEXT NOT NULL,
+    num_attempts INT NOT NULL,
+    last_attempt_ts BIGINT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS event_backfill_attempts_event_id ON event_backfill_attempts(event_id);