speedups (#8)

config/game_1920_1080.ini

@@ -9,8 +9,12 @@ window_dimensions=1920,1080
 
 [ui_offsets]
 ; all offsets in [x, y]
-item_descr=387,-1
-equip_to_bottom=-1,25
+item_descr_width=387
+item_descr_pad=15
+item_descr_off_bottom_edge=52
+find_seperator_short_offset_top=300
+find_bullet_points_width=39
+item_descr_line_height=25
 
 [ui_roi]
 character_active=1306,13,154,30

src/config.py

@@ -86,7 +86,7 @@ def load_data(self):
             self.ui_pos[key] = tuple(int(val) for val in self._select_val("ui_pos", key).split(","))
 
         for key in self.configs["game"]["parser"]["ui_offsets"]:
-            self.ui_offsets[key] = np.array([int(x) for x in self._select_val("ui_offsets", key).split(",")])
+            self.ui_offsets[key] = int(self._select_val("ui_offsets", key))
 
         for key in self.configs["game"]["parser"]["ui_roi"]:
             self.ui_roi[key] = np.array([int(x) for x in self._select_val("ui_roi", key).split(",")])

src/item/data/affix.py

@@ -4,5 +4,5 @@
 @dataclass
 class Affix:
     type: str
-    text: str
     value: float = None
+    text: str = ""

src/item/data/aspect.py

@@ -4,5 +4,5 @@
 @dataclass
 class Aspect:
     type: str
-    text: str
     value: float = None
+    text: str = ""

src/item/find_descr.py

@@ -2,7 +2,7 @@
 from copy import copy
 from item.data.rarity import ItemRarity
 from config import Config
-from template_finder import SearchArgs
+from template_finder import search
 from utils.image_operations import crop
 from utils.roi_operations import fit_roi_to_window_size
 
@@ -17,7 +17,8 @@
 
 
 def find_descr(img: np.ndarray, anchor: tuple[int, int]) -> tuple[bool, tuple[int, int], ItemRarity, np.ndarray]:
-    item_descr_width, _ = Config().ui_offsets["item_descr"]
+    item_descr_width = Config().ui_offsets["item_descr_width"]
+    item_descr_pad = Config().ui_offsets["item_descr_pad"]
     _, window_height = Config().ui_pos["window_dimensions"]
 
     refs = list(map_template_rarity.keys())
@@ -27,30 +28,30 @@ def find_descr(img: np.ndarray, anchor: tuple[int, int]) -> tuple[bool, tuple[in
     roi_left[0] += anchor[0]
     ok, roi_left = fit_roi_to_window_size(roi_left, Config().ui_pos["window_dimensions"])
     if ok:
-        res = SearchArgs(ref=refs, roi=roi_left, threshold=0.93, mode="best").detect()
+        res = search(ref=refs, inp_img=img, roi=roi_left, threshold=0.93, mode="best")
     if res is not None and not res.success:
         roi_right = copy(Config().ui_roi["rel_descr_search_right"])
         roi_right[0] += anchor[0]
         ok, roi_right = fit_roi_to_window_size(roi_right, Config().ui_pos["window_dimensions"])
         if ok:
-            res = SearchArgs(ref=refs, roi=roi_right, threshold=0.93, mode="best").detect()
+            res = search(ref=refs, inp_img=img, roi=roi_right, threshold=0.93, mode="best")
 
     if res is not None and res.success:
         match = res.matches[0]
         rarity = map_template_rarity[match.name.lower()]
         # find equipe template
-        equip_roi = [match.region[0] - 30, match.region[1], item_descr_width, window_height]
-        res_bottom = SearchArgs(ref=["item_descr_equip", "item_descr_equip_inactive"], roi=equip_roi, threshold=0.78).detect()
-        if not res_bottom.success:
-            res_bottom = SearchArgs(ref=["item_shift_link"], roi=equip_roi, threshold=0.78).detect()
-
+        offset_top = int(window_height * 0.1)
+        roi_y = match.region[1] - offset_top
+        search_height = window_height - roi_y - offset_top
+        roi = [match.region[0], roi_y, item_descr_width, search_height]
+        res_bottom = search(ref=["item_bottom_edge"], inp_img=img, roi=roi, threshold=0.73, mode="best")
         if res_bottom.success:
-            _, off_bottom_of_descr = Config().ui_offsets["equip_to_bottom"]
+            off_bottom_of_descr = Config().ui_offsets["item_descr_off_bottom_edge"]
             equip_match = res_bottom.matches[0]
             crop_roi = [
-                match.region[0],
-                match.region[1],
-                item_descr_width,
+                match.region[0] + item_descr_pad,
+                match.region[1] + item_descr_pad,
+                item_descr_width - 2 * item_descr_pad,
                 equip_match.center[1] - off_bottom_of_descr - match.region[1],
             ]
             croped_descr = crop(img, crop_roi)

src/item/models.py

@@ -12,3 +12,23 @@ class Item:
     power: int | None = None
     aspect: Aspect | None = None
     affixes: list[Affix] = field(default_factory=list)
+
+    def __eq__(self, other):
+        if not isinstance(other, Item):
+            return False
+
+        if self.aspect is None and other.aspect is not None:
+            return False
+        if self.aspect is not None and other.aspect is None:
+            return False
+        if self.aspect is not None and other.aspect is not None:
+            if self.aspect.type != other.aspect.type or self.aspect.value != other.aspect.value:
+                return False
+
+        return (
+            self.rarity == other.rarity
+            and self.power == other.power
+            and self.type == other.type
+            and len(self.affixes) == len(other.affixes)
+            and all(s.type == o.type and s.value == o.value for s, o in zip(self.affixes, other.affixes))
+        )

src/item/read_descr.py

@@ -1,4 +1,5 @@
 import numpy as np
+import time
 import cv2
 from logger import Logger
 from item.data.rarity import ItemRarity
@@ -13,6 +14,7 @@
 import re
 import json
 from rapidfuzz import process
+from config import Config
 
 affix_dict = dict()
 with open("assets/affixes.json", "r") as f:
@@ -61,7 +63,7 @@ def _clean_str(s):
     cleaned_str = re.sub(
         r"\((rogue|barbarian|druid|sorcerer|necromancer) only\)", "", cleaned_str
     )  # this is not included in our affix table
-    cleaned_str = _remove_text_after_first_keyword(cleaned_str, ["requires level", "account", "sell value"])
+    cleaned_str = _remove_text_after_first_keyword(cleaned_str, ["requires level", "requires lev", "account", "sell value"])
     cleaned_str = re.sub(
         r"(scroll up|account bound|requires level|sell value|durability|barbarian|rogue|sorceress|druid|necromancer|not useable|by your class|by your clas)",
         "",
@@ -73,34 +75,27 @@ def _clean_str(s):
 
 def read_descr(rarity: ItemRarity, img_item_descr: np.ndarray) -> Item:
     item = Item(rarity)
+    img_height, img_width, _ = img_item_descr.shape
+    line_height = Config().ui_offsets["item_descr_line_height"]
 
     # Detect textures (1)
     # =====================================
-    refs = ["item_seperator_long", "item_seperator_long_2"]
-    if not (
-        seperator_long := search(refs, img_item_descr, threshold=0.85, use_grayscale=True, mode="all", color_match="gray_seperator")
-    ).success:
-        Logger.warning("Could not detect item_seperator_long.")
-        screenshot("failed_seperator_long", img=img_item_descr)
-        return None
-    seperator_long.matches = sorted(seperator_long.matches, key=lambda match: match.center[1])
-    # Mask img where seperator_long was found
-    masked_search_img = img_item_descr.copy()
-    for match in seperator_long.matches:
-        x, y, w, h = match.region
-        cv2.rectangle(masked_search_img, (x, y), (x + w, y + h), (0, 0, 0), -1)
-    refs = ["item_seperator_short", "item_seperator_short_2"]
-    if not (
-        seperator_short := search(refs, masked_search_img, threshold=0.68, use_grayscale=True, mode="best", color_match="gray_seperator")
-    ).success:
+    start_tex_1 = time.time()
+    refs = ["item_seperator_short_rare", "item_seperator_short_legendary"]
+    roi = [0, 0, img_item_descr.shape[1], Config().ui_offsets["find_seperator_short_offset_top"]]
+    if not (sep_short := search(refs, img_item_descr, 0.68, roi, True, "gray_seperator", "all")).success:
         Logger.warning("Could not detect item_seperator_short.")
-        screenshot("failed_seperator_short", img=masked_search_img)
+        screenshot("failed_seperator_short", img=img_item_descr)
         return None
+    sorted_matches = sorted(sep_short.matches, key=lambda match: match.center[1])
+    sep_short_match = sorted_matches[0]
+    # print("-----")
+    # print("Runtime (start_tex_1): ", time.time() - start_tex_1)
 
     # Item Type and Item Power
     # =====================================
-    _, w, _ = img_item_descr.shape
-    roi_top = [15, 15, w - 30, seperator_short.matches[0].center[1] - 20]
+    start_power = time.time()
+    roi_top = [0, 0, img_width, sep_short_match.center[1]]
     crop_top = crop(img_item_descr, roi_top)
     concatenated_str = image_to_text(crop_top).text.lower().replace("\n", " ")
     idx = None
@@ -133,47 +128,58 @@ def read_descr(rarity: ItemRarity, img_item_descr: np.ndarray) -> Item:
         Logger().warning(f"Could not detect ItemPower and ItemType: {concatenated_str}")
         screenshot("failed_itempower_itemtype", img=img_item_descr)
         return None
+    # print("Runtime (start_power): ", time.time() - start_power)
 
     # Detect textures (2)
     # =====================================
-    if item.type in [ItemType.Helm, ItemType.Armor, ItemType.Gloves]:
-        roi_bullets = [0, seperator_short.matches[0].center[1], 100, 1080]
-    else:
-        roi_bullets = [0, seperator_long.matches[0].center[1], 100, 1080]
-    if not (
-        affix_bullets := search("affix_bullet_point", img_item_descr, threshold=0.87, roi=roi_bullets, use_grayscale=True, mode="all")
-    ).success:
+    start_tex_2 = time.time()
+    roi_bullets = [0, sep_short_match.center[1], Config().ui_offsets["find_bullet_points_width"], img_height]
+    if not (affix_bullets := search("affix_bullet_point", img_item_descr, 0.87, roi_bullets, True, mode="all")).success:
         Logger.warning("Could not detect affix_bullet_points.")
         screenshot("failed_affix_bullet_points", img=img_item_descr)
         return None
     affix_bullets.matches = sorted(affix_bullets.matches, key=lambda match: match.center[1])
-    empty_sockets = search("empty_socket", img_item_descr, threshold=0.87, roi=roi_bullets, use_grayscale=True, mode="all")
+    # Depending on the item type we have to remove some of the topmost affixes as they are fixed
+    remove_top_most = 1
+    if item.type in [ItemType.Armor, ItemType.Helm, ItemType.Gloves]:
+        remove_top_most = 0
+    elif item.type in [ItemType.Ring]:
+        remove_top_most = 2
+    elif item.type in [ItemType.Shield]:
+        remove_top_most = 4
+    else:
+        # default for: Pants, Amulets, Boots, All Weapons
+        remove_top_most = 1
+    affix_bullets.matches = affix_bullets.matches[remove_top_most:]
+    empty_sockets = search("empty_socket", img_item_descr, 0.87, roi_bullets, True, mode="all")
     empty_sockets.matches = sorted(empty_sockets.matches, key=lambda match: match.center[1])
-    aspect_bullets = search("aspect_bullet_point", img_item_descr, threshold=0.87, roi=roi_bullets, use_grayscale=True, mode="first")
+    aspect_bullets = search("aspect_bullet_point", img_item_descr, 0.87, roi_bullets, True, mode="first")
     if rarity == ItemRarity.Legendary and not aspect_bullets.success:
         Logger.warning("Could not detect aspect_bullet for a legendary item.")
         screenshot("failed_aspect_bullet", img=img_item_descr)
         return None
+    # print("Runtime (start_tex_2): ", time.time() - start_tex_2)
 
     # Affixes
     # =====================================
+    start_affix = time.time()
     # Affix starts at first bullet point
-    affix_start = [affix_bullets.matches[0].center[0] + 7, affix_bullets.matches[0].center[1] - 16]
-    # Affix ends at aspect bullet, empty sockets or seperator line
+    affix_start = [affix_bullets.matches[0].center[0] + line_height // 4, affix_bullets.matches[0].center[1] - int(line_height * 0.7)]
+    # Affix ends at aspect bullet or empty sockets
+    bottom_limit = 0
     if rarity == ItemRarity.Legendary:
         bottom_limit = aspect_bullets.matches[0].center[1]
     elif len(empty_sockets.matches) > 0:
         bottom_limit = empty_sockets.matches[0].center[1]
-    else:
-        bottom_limit = seperator_long.matches[-1].center[1]
     if bottom_limit < affix_start[1]:
-        bottom_limit = img_item_descr.shape[0]
+        bottom_limit = img_height
     # Calc full region of all affixes
-    affix_width = w - affix_start[0] - 30
-    affix_height = bottom_limit - affix_start[1] - 7
+    affix_width = img_width - affix_start[0]
+    affix_height = bottom_limit - affix_start[1] - int(line_height * 0.4)
     full_affix_region = [*affix_start, affix_width, affix_height]
-    cropp_full_affix = crop(img_item_descr, full_affix_region)
-    affix_lines = image_to_text(cropp_full_affix).text.lower().split("\n")
+    crop_full_affix = crop(img_item_descr, full_affix_region)
+    # cv2.imwrite("crop_full_affix.png", crop_full_affix)
+    affix_lines = image_to_text(crop_full_affix).text.lower().split("\n")
     affix_lines = [line for line in affix_lines if line]  # remove empty lines
     # split affix text based on distance of affix bullet points
     delta_y_arr = [
@@ -185,10 +191,10 @@ def read_descr(rarity: ItemRarity, img_item_descr: np.ndarray) -> Item:
         if dy is None:
             combined_lines = "\n".join(affix_lines[line_idx:])
         else:
-            closest_value = _closest_to(dy, [25, 50, 75])
-            if closest_value == 25:
+            closest_value = _closest_to(dy, [line_height, line_height * 2, line_height * 3])
+            if closest_value == line_height:
                 lines_to_add = 1
-            elif closest_value == 50:
+            elif closest_value == line_height * 2:
                 lines_to_add = 2
             else:  # closest_value == 75
                 lines_to_add = 3
@@ -201,35 +207,38 @@ def read_descr(rarity: ItemRarity, img_item_descr: np.ndarray) -> Item:
         found_value = _find_number(combined_lines)
 
         if found_key is not None:
-            item.affixes.append(Affix(found_key, combined_lines, found_value))
+            item.affixes.append(Affix(found_key, found_value, combined_lines))
         else:
             Logger.warning(f"Could not find affix: {cleaned_str}")
             screenshot("failed_affixes", img=img_item_descr)
             return None
+    # print("Runtime (start_affix): ", time.time() - start_affix)
 
     # Aspect
     # =====================================
+    start_aspect = time.time()
     if rarity == ItemRarity.Legendary:
         ab = aspect_bullets.matches[0].center
-        bottom_limit = empty_sockets.matches[0].center[1] if len(empty_sockets.matches) > 0 else seperator_long.matches[-1].center[1]
-        # in case of scroll down is visible the bottom seperator is not visible
-        if bottom_limit < ab[1]:
-            bottom_limit = img_item_descr.shape[0]
+        bottom_limit = empty_sockets.matches[0].center[1] if len(empty_sockets.matches) > 0 else img_height
+        dx_offset = line_height // 4
+        dy_offset = int(line_height * 0.7)
         dy = bottom_limit - ab[1]
-        roi_full_aspect = [ab[0] + 7, max(0, ab[1] - 16), w - 30 - ab[0], dy]
+        roi_full_aspect = [ab[0] + dx_offset, ab[1] - dy_offset, img_width - ab[0] - dx_offset - 1, dy]
         img_full_aspect = crop(img_item_descr, roi_full_aspect)
+        # cv2.imwrite("img_full_aspect.png", img_full_aspect)
         concatenated_str = image_to_text(img_full_aspect).text.lower().replace("\n", " ")
         cleaned_str = _clean_str(concatenated_str)
 
         found_key = _closest_match(cleaned_str, aspect_dict, min_score=77)
-        idx = 1 if found_key in ["frostbitten_aspect"] else 0
+        idx = 1 if found_key in ["frostbitten_aspect", "aspect_of_artful_initiative"] else 0
         found_value = _find_number(concatenated_str, idx)
 
         if found_key is not None:
-            item.aspect = Aspect(found_key, concatenated_str, found_value)
+            item.aspect = Aspect(found_key, found_value, concatenated_str)
         else:
             Logger.warning(f"Could not find aspect: {cleaned_str}")
             screenshot("failed_aspect", img=img_item_descr)
             return None
+    # print("Runtime (start_aspect): ", time.time() - start_aspect)
 
     return item

src/utils/ocr/read.py

@@ -7,6 +7,8 @@
 TESSDATA_PATH = "assets/tessdata"
 
 API = PyTessBaseAPI(psm=3, oem=OEM.LSTM_ONLY, path=TESSDATA_PATH, lang="eng")
+# supposed to give fruther runtime improvements, but reading performance really goes down...
+# API.SetVariable("tessedit_do_invert", "0")
 
 
 def _img_to_bytes(image: np.ndarray, colorspace: str = "BGR"):

test/item/find_descr_test.py

@@ -0,0 +1,31 @@
+import time
+import pytest
+import cv2
+from item.find_descr import find_descr
+from item.data.rarity import ItemRarity
+from cam import Cam
+
+
+BASE_PATH = "test/assets/item"
+
+
+@pytest.mark.parametrize(
+    "img_res, input_img, anchor, expected_success, expected_top_left, expected_rarity",
+    [
+        ((1920, 1080), f"{BASE_PATH}/find_descr_rare_1920x1080.png", (1630, 763), True, (1196, 377), ItemRarity.Rare),
+        ((1920, 1080), f"{BASE_PATH}/find_descr_legendary_1920x1080.png", (1515, 761), True, (1088, 78), ItemRarity.Legendary),
+    ],
+)
+def test_find_descr(img_res, input_img, anchor, expected_success, expected_top_left, expected_rarity):
+    Cam().update_window_pos(0, 0, img_res[0], img_res[1])
+    img = cv2.imread(input_img)
+    start = time.time()
+    success, top_left_corner, item_rarity, cropped_img = find_descr(img, anchor)
+    print("Runtime (find_descr()): ", time.time() - start)
+    if success and False:
+        cv2.imwrite(f"item_descr.png", cropped_img)
+    assert success == expected_success
+    tolerance = 0.01 * img_res[0]
+    assert abs(top_left_corner[0] - expected_top_left[0]) <= tolerance
+    assert abs(top_left_corner[1] - expected_top_left[1]) <= tolerance
+    assert item_rarity == expected_rarity