A DB upgrade that cleans erroneous health info from the DB

src/tribler/core/components/metadata_store/db/store.py

@@ -54,7 +54,7 @@
 from tribler.core.utilities.utilities import MEMORY_DB
 
 BETA_DB_VERSIONS = [0, 1, 2, 3, 4, 5]
-CURRENT_DB_VERSION = 14
+CURRENT_DB_VERSION = 15
 
 MIN_BATCH_SIZE = 10
 MAX_BATCH_SIZE = 1000

src/tribler/core/upgrade/tests/test_upgrader.py

@@ -1,5 +1,8 @@
 import os
+import random
 import shutil
+import time
+from binascii import unhexlify
 from pathlib import Path
 from typing import Set
 from unittest.mock import patch
@@ -194,6 +197,60 @@ def _exists(db, table, column):
         assert mds.get_value('db_version') == '14'
 
 
+def test_upgrade_pony14to15(upgrader: TriblerUpgrader, channels_dir, trustchain_keypair, mds_path):
+    _copy(source_name='pony_v14.db', target=mds_path)
+
+    now = int(time.time())
+    in_the_past = now - 1000
+    in_the_future = now + 1000
+    mds = MetadataStore(mds_path, channels_dir, trustchain_keypair, check_tables=False)
+
+    def _add_torrent_state(self_checked, last_check):
+        mds.TorrentState(infohash=random.randbytes(20), seeders=1, leechers=1,
+                         self_checked=self_checked, last_check=last_check)
+
+    with db_session:
+        mds.TorrentState(infohash=random.randbytes(20))  # a TorrentState for an infohash that was never checked
+        _add_torrent_state(self_checked=0, last_check=in_the_past)
+        _add_torrent_state(self_checked=1, last_check=in_the_past)
+        _add_torrent_state(self_checked=0, last_check=in_the_future)
+        _add_torrent_state(self_checked=1, last_check=in_the_future)
+
+    def _execute(sql, **kwargs):
+        return mds.db.execute(sql, kwargs).fetchone()[0]
+
+    with db_session:
+        assert mds.get_value('db_version') == '14'
+        # Total number of records should not be changed after the upgrade
+        assert _execute('select count(*) from TorrentState') == 5
+        # There will be fewer records with nonzero seeders/leechers after the upgrade
+        assert _execute('select count(*) from TorrentState where seeders > 0 or leechers > 0') == 4
+        # Before the upgrade, the database contained several corrupted records, and SQL queries were able to find them
+        assert _execute('select count(*) from TorrentState where self_checked > 0') == 2
+        assert _execute('select count(*) from TorrentState where last_check > $x', x=now) == 2
+
+    mds.shutdown()
+
+    # The upgrade should clear the self_checked flag for all records, as due to a bug, we cannot be sure they are
+    # really self-checked. Also, it should clear all records with the future last_check timestamp value, resetting
+    # their seeders/leechers values
+    upgrader.upgrade_pony_db_14to15()
+
+    mds = MetadataStore(mds_path, channels_dir, trustchain_keypair, check_tables=False)
+    with db_session:
+        assert mds.get_value('db_version') == '15'
+        # After the upgrade, the same SQL queries found the same total number of records
+        assert _execute('select count(*) from TorrentState') == 5
+        # Records with correct last_check values still have their seeders/leechers values;
+        # only the records with incorrect last_check values were cleared
+        assert _execute('select count(*) from TorrentState where seeders > 0 or leechers > 0') == 2
+        # After the upgrade, the same SQL queries found no corrupted records
+        assert _execute('select count(*) from TorrentState where self_checked > 0') == 0
+        assert _execute('select count(*) from TorrentState where last_check > $x', x=now) == 0
+
+    mds.shutdown()
+
+
 def test_upgrade_pony12to13(upgrader, channels_dir, mds_path, trustchain_keypair):  # pylint: disable=W0621
     _copy('pony_v12.db', mds_path)
 

src/tribler/core/upgrade/upgrade.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import shutil
+import time
 from configparser import MissingSectionHeaderError, ParsingError
 from types import SimpleNamespace
 from typing import List, Optional, Tuple
@@ -102,6 +103,7 @@ def run(self):
         self.upgrade_pony_db_13to14()
         self.upgrade_tags_to_knowledge()
         self.remove_old_logs()
+        self.upgrade_pony_db_14to15()
 
     def remove_old_logs(self) -> Tuple[List[Path], List[Path]]:
         self._logger.info(f'Remove old logs')
@@ -128,6 +130,16 @@ def upgrade_tags_to_knowledge(self):
         migration = MigrationTagsToKnowledge(self.state_dir, self.secondary_key)
         migration.run()
 
+    def upgrade_pony_db_14to15(self):
+        mds_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db'
+
+        mds = MetadataStore(mds_path, self.channels_dir, self.primary_key, disable_sync=True,
+                            check_tables=False, db_version=14) if mds_path.exists() else None
+
+        self.do_upgrade_pony_db_14to15(mds)
+        if mds:
+            mds.shutdown()
+
     def upgrade_pony_db_13to14(self):
         mds_path = self.state_dir / STATEDIR_DB_DIR / 'metadata.db'
         tagdb_path = self.state_dir / STATEDIR_DB_DIR / 'tags.db'
@@ -255,6 +267,30 @@ def do_upgrade_pony_db_12to13(self, mds):
 
             db_version.value = str(to_version)
 
+    def do_upgrade_pony_db_14to15(self, mds: Optional[MetadataStore]):
+        if not mds:
+            return
+
+        version = SimpleNamespace(current='14', next='15')
+        with db_session:
+            db_version = mds.get_value(key='db_version')
+            if db_version != version.current:
+                return
+
+            self._logger.info('Clean the incorrectly set self_checked flag for health info in the db')
+            sql = 'UPDATE "TorrentState" SET "self_checked" = 0 WHERE "self_checked" != 0;'
+            cursor = mds.db.execute(sql)
+            self._logger.info(f'The self_checked flag was cleared in {cursor.rowcount} rows')
+
+            self._logger.info('Reset the last_check future timestamps values to zero')
+            now = int(time.time())  # pylint: disable=unused-variable
+            sql = 'UPDATE "TorrentState" SET "seeders" = 0, "leechers" = 0, "has_data" = 0, "last_check" = 0 ' \
+                  ' WHERE "last_check" > $now;'
+            cursor = mds.db.execute(sql)
+            self._logger.info(f'{cursor.rowcount} rows with future last_check timestamps were reset')
+
+            mds.set_value(key='db_version', value=version.next)
+
     def do_upgrade_pony_db_13to14(self, mds: Optional[MetadataStore], tags: Optional[TagDatabase]):
         def add_column(db, table_name, column_name, column_type):
             if not self.column_exists_in_table(db, table_name, column_name):