New database design

* Use database relations instead of duplicating data.
* Move database queries to db module

Since the query for snatches returns tuples it is a bit more
inconvienient when using the data. But for the greater good is should be
fine. Converting to named tuple or similar seems like a waste.

Convert script included to read the old design from one file and write
the new design to another one.

convert_db.py

@@ -0,0 +1,85 @@
+#!/usr/bin/python3
+import datetime
+import logging
+from pony.orm import *
+import os
+import sys
+
+# This script converts a database file from the old database design to the new one.
+# It takes two arguments. The old brain.db and new database to be created.
+# E.g. ./convert_db.py ~/.arrnounced/brain.db new_brain.db
+# Copy the old file for safe keeping and replace it with the new file.
+# N.B. Since this script converts from old to new design in one single database
+# session it consumes a fair bit of memory.
+
+def def_new(db):
+    class Announced(db.Entity):
+        date = Required(datetime.datetime)
+        title = Required(str)
+        indexer = Required(str)
+        torrent = Required(str)
+        backend = Required(str)
+        snatched = Set('Snatched')
+
+    class Snatched(db.Entity):
+        date = Required(datetime.datetime)
+        announced = Required(Announced)
+        backend = Required(str)
+
+def def_old(db):
+    class Announced(db.Entity):
+        date = Required(datetime.datetime)
+        title = Required(str)
+        indexer = Required(str)
+        torrent = Required(str)
+        backend = Required(str)
+
+    class Snatched(db.Entity):
+        date = Required(datetime.datetime)
+        title = Required(str)
+        indexer = Required(str)
+        torrent = Required(str)
+        backend = Required(str)
+
+def open_database(filename, definition, create_db):
+    db = Database()
+    definition(db)
+    db.bind('sqlite', os.path.realpath(filename), create_db=create_db)
+    db.generate_mapping(create_tables=create_db)
+    return db
+
+if len(sys.argv) != 3:
+    print("Usage: " + sys.argv[0] + " <old database> <new database>")
+    sys.exit(1)
+
+if os.path.isfile(sys.argv[2]):
+    print("Error: New database cannot be existing file")
+    sys.exit(1)
+
+old_db = open_database(sys.argv[1], def_old, False)
+new_db = open_database(sys.argv[2], def_new, True)
+
+with db_session:
+    old_anns = old_db.Announced.select().order_by(old_db.Announced.id)
+    for old_ann in old_anns:
+        new_ann = new_db.Announced(date=old_ann.date,
+                title=old_ann.title,
+                indexer=old_ann.indexer,
+                torrent=old_ann.torrent,
+                backend=old_ann.backend)
+
+        old_sns = list(select(s for s in old_db.Snatched if s.title == old_ann.title))
+        #old_sns.show()
+        if len(old_sns) == 0:
+            #print(old_ann.title + " was not snatched")
+            continue
+        elif len(old_sns) > 1:
+            print("Warning: Found more than once snatch which matched an announcment. Choosing first one")
+
+        old_sn = old_sns[0]
+        new_db.Snatched(date=old_sn.date,
+                announced=new_ann,
+                backend=old_sn.backend)
+
+    #new_db.Announced.select().order_by(new_db.Announced.id).show()
+    #new_db.Snatched.select().order_by(new_db.Snatched.id).show()

src/backend.py

@@ -49,7 +49,6 @@ def _string_to_backend(backend_name):
             return backend
     return None
 
-
 def backends_to_string(backends):
     return "/".join(_backend_data[x]['name'] for x in backends) if len(backends) > 0 else "None"
 

src/db.py

@@ -13,16 +13,40 @@ class Announced(db.Entity):
     indexer = Required(str)
     torrent = Required(str)
     backend = Required(str)
+    snatched = Set('Snatched')
 
 
 class Snatched(db.Entity):
     date = Required(datetime.datetime)
-    title = Required(str)
-    indexer = Required(str)
-    torrent = Required(str)
+    announced = Required(Announced)
     backend = Required(str)
 
-
 def init(destination_dir):
     db.bind('sqlite', os.path.join(os.path.realpath(destination_dir), "brain.db"), create_db=True)
     db.generate_mapping(create_tables=True)
+
+def get_snatched(limit, page):
+    # Order by first attribute in tuple i.e. s.id
+    ss = pony.orm.select(
+            (s.id, s.date, a.indexer, a.title, s.backend)
+            for s in Snatched for a in s.announced
+            ).order_by(desc(1)).limit(limit, offset=page*limit)
+    return ss
+
+def get_announced(limit, page):
+    return Announced.select().order_by(desc(Announced.id)).limit(limit, offset=page*limit)
+
+def get_announcement(id):
+    return Announced.get(id=id)
+
+def insert_announcement(announcement, indexer, backends):
+    return Announced(date=datetime.datetime.now(),
+            title=announcement.torrent_name,
+            torrent=announcement.torrent_url,
+            indexer=indexer,
+            backend=backends)
+
+def insert_snatched(announcement, backend):
+    Snatched(date=datetime.datetime.now(),
+            announced=announcement,
+            backend=backend)

src/message_handler.py

@@ -1,5 +1,4 @@
 import asyncio
-import datetime
 import html
 import logging
 import re
@@ -45,8 +44,7 @@ async def on_message(tracker_config, source, target, message):
                 tracker_config.short_name, tracker_config.announce_delay, announcement.torrent_name)
         await asyncio.sleep(tracker_config.announce_delay)
 
-    db.Announced(date=datetime.datetime.now(), title=announcement.torrent_name,
-            indexer=tracker_config.short_name, torrent=announcement.torrent_url, backend=backends_string)
+    db_announced = db.insert_announcement(announcement, tracker_config.short_name, backends_string)
     logger.info("Notifying %s of release from %s: %s", backends_string,
                 tracker_config.short_name, announcement.torrent_name)
 
@@ -57,5 +55,4 @@ async def on_message(tracker_config, source, target, message):
         logger.debug("Release was rejected: %s", announcement.torrent_name)
     else:
         logger.info("%s approved release: %s", backend, announcement.torrent_name)
-        snatched = db.Snatched(date=datetime.datetime.now(), title=announcement.torrent_name,
-                indexer=tracker_config.short_name, torrent=announcement.torrent_url, backend=backend)
+        db.insert_snatched(db_announced, backend)

src/templates/index.html

@@ -47,10 +47,10 @@
                         <tbody>
                         {% for snatch in snatched %}
                         <tr>
-                            <td>{{ format_timestamp(snatch.date) }}</td>
-                            <td>{{ snatch.indexer }}</td>
-                            <td>{{ snatch.title }}</td>
-                            <td>{{ snatch.backend }}</td>
+                            <td>{{ format_timestamp(snatch[1]) }}</td>
+                            <td>{{ snatch[2] }}</td>
+                            <td>{{ snatch[3] }}</td>
+                            <td>{{ snatch[4] }}</td>
                         </tr>
                         {% endfor %}
                         </tbody>

src/webui.py

@@ -115,8 +115,8 @@ def send_asset(path):
 @login_required
 @db.db_session
 def index():
-    return render_template('index.html', snatched=db.Snatched.select().order_by(db.desc(db.Snatched.date)).limit(20),
-                           announced=db.Announced.select().order_by(db.desc(db.Announced.date)).limit(20),
+    return render_template('index.html', snatched=db.get_snatched(limit=20, page=0),
+                           announced=db.get_announced(limit=20, page=0),
                            backends=get_configured_backends())
 
 @app.route("/logs")
@@ -160,7 +160,7 @@ def notify():
         data = request.json
         if 'id' in data and 'backend_name' in data:
             # Request to check this torrent again
-            announcement = db.Announced.get(id=data.get('id'))
+            announcement = db.get_announcement(data.get('id'))
             if announcement is not None and len(announcement.title) > 0:
                 logger.debug("Checking announcement again: %s", announcement.title)
 
@@ -169,6 +169,7 @@ def notify():
 
                 if approved:
                     logger.debug(backend_name + " accepted the torrent this time!")
+                    db.insert_snatched(announcement, backend_name)
                     return "OK"
 
                 logger.debug(backend_name + " still refused this torrent...")