Scrape Wayback Machine instead of J-Archive (w/ fallback)

jparty/borders.py

@@ -48,10 +48,10 @@ def __flash_hints(self, key):
         while self.__active_thread == current_thread():
             for b in self:
                 b.show_hints(key)
-            time.sleep(0.5)
-            for b in self:
-                b.hide_hints(key)
-            time.sleep(0.5)
+            #time.sleep(0.5)
+            #for b in self:
+             #   b.hide_hints(key)
+            #time.sleep(0.5)
 
     def buzz_hint(self):
         self.__buzz_hint_thread = Thread(target=self.__buzz_hint, name="buzz_hint")

jparty/controller.py

@@ -152,8 +152,16 @@ def __init__(self, game):
         self.connected_players = []
         self.accepting_players = True
 
-    def start(self, threaded=True):
-        self.app.listen(self.port)
+    def start(self, threaded=True, tries=0):
+        try:
+            self.app.listen(self.port)
+        except OSError as e:
+            if tries>10:
+                raise Exception("Cannot find open port")
+            self.port += 1
+            self.start(threaded, tries+1)
+            return
+
         if threaded:
             self.thread = Thread(target=tornado.ioloop.IOLoop.current().start)
             self.thread.setDaemon(True)

jparty/main.py

@@ -84,6 +84,12 @@ def main():
 
     game.setBuzzerController(socket_controller)
 
+    try:
+        socket_controller.start()
+    except PermissionError as e:
+        permission_error()
+        exit(1)
+
     main_window = DisplayWindow(game)
     host_window = HostDisplayWindow(game)
     game.setDisplays(host_window, main_window)
@@ -96,11 +102,6 @@ def main():
 
     song_player = game.song_player
 
-    try:
-        socket_controller.start()
-    except PermissionError as e:
-        permission_error()
-        exit(1)
 
 
     r=1 # fail by default

jparty/retrieve.py

@@ -1,6 +1,8 @@
 import requests
 from bs4 import BeautifulSoup
+from html import unescape
 import re
+import json
 from jparty.game import Question, Board, FinalBoard, GameData
 import logging
 import csv
@@ -50,14 +52,17 @@ def get_Gsheet_game(file_id):
 
 def get_game(game_id):
     if len(str(game_id)) < 7:
-        return get_JArchive_Game(game_id)
+        return get_wayback_jarchive_game(game_id)
     else:
         return get_Gsheet_game(str(game_id))
 
 
-def get_JArchive_Game(game_id, soup=None, is_wayback=False, wayback_url=None):
+def findanswer(clue):
+    return re.findall(r'correct_response">(.*?)</em', unescape(str(clue)))[0]
+
+def get_JArchive_Game(game_id, wayback_url=None):
     logging.info(f"getting game {game_id}")
-    if is_wayback:
+    if wayback_url is not None:
         r = requests.get(wayback_url)
     else:
         r = requests.get(f"http://www.j-archive.com/showgame.php?game_id={game_id}")
@@ -92,30 +97,54 @@ def get_JArchive_Game(game_id, soup=None, is_wayback=False, wayback_url=None):
             )  # get index from id string
             dd = clue.find(class_="clue_value_daily_double") is not None
             value = MONIES[i][index[1]]
-            answer = re.findall(r'correct_response">(.*?)</em', str(clue))[0]
+            answer = findanswer(clue)
             questions.append(
                 Question(index, text, answer, categories[index[0]], value, dd)
             )
         boards.append(Board(categories, questions, dj=(i == 1)))
 
     # Final Jeopardy
-    fro = soup.find_all(class_="final_round")[0]
-    category_obj = fro.find_all(class_="category")[0]
+    final_round_obj = soup.find_all(class_="final_round")[0]
+    category_obj = final_round_obj.find_all(class_="category")[0]
     category = category_obj.find(class_="category_name").text
-    clue = fro.find_all(class_="clue")[0]
+    clue = final_round_obj.find_all(class_="clue")[0]
     text_obj = clue.find(class_="clue_text")
     if text_obj is None:
         logging.info("this game is incomplete")
         return None
 
     text = text_obj.text
-    answer = re.findall(r'correct_response">(.*)</em', str(clue))[0]
+    answer = findanswer(final_round_obj)
     question = Question((0, 0), text, answer, category)
 
     boards.append(FinalBoard(category, question))
 
     return GameData(boards, date, comments)
 
+def get_wayback_jarchive_game(game_id):
+    # kudos to Abhi Kumbar: https://medium.com/analytics-vidhya/the-wayback-machine-scraper-63238f6abb66
+    # this query's the wayback cdx api for possible instances of the saved jarchive page with the specified game id & returns the latest one
+    JArchive_url = f"j-archive.com/showgame.php?game_id={str(game_id)}"  # use the url w/o the http:// or https:// to include both in query
+    url = f'http://web.archive.org/cdx/search/cdx?url={JArchive_url}&collapse=digest&limit=-2&fastLatest=true&output=json'  # for some reason, using limit=-1 does not work
+    urls = requests.get(url).text
+    parse_url = json.loads(urls)  # parses the JSON from urls.
+    if len(parse_url) == 0:  # if no results, return None
+        logging.info("no games found in wayback")
+        # return None
+        # alternative: use fallback to get game from scraping j-archive directly
+        return get_JArchive_Game(game_id)
+
+    ## Extracts timestamp and original columns from urls and compiles a url list.
+    url_list = []
+    for i in range(1, len(parse_url)): # gets the wayback url
+        orig_url = parse_url[i][2]
+        tstamp = parse_url[i][1]
+        waylink = tstamp + '/' + orig_url
+        final_url = f'http://web.archive.org/web/{waylink}'
+        url_list.append(final_url)
+    latest_url = url_list[-1]
+    return get_JArchive_Game(game_id, latest_url)
+
 def get_game_sum(soup):
     date = re.search(
         r"- \w+, (.*?)$", soup.select("#game_title > h1")[0].contents[0]

requirements.txt

@@ -1,7 +1,7 @@
 PyQt6==6.4.0
-requests==2.28.1
+requests==2.31.0
 simpleaudio==1.0.4
-tornado==6.2
+tornado==6.3.2
 BeautifulSoup4==4.11.1
 pyinstaller==5.6.2
 qrcode==7.3.1