feat/restricted-mouse-clicks (aeon0#709)

* feat/restricted-mouse-clicks

* fix typos, use GoldBtnInventory ScreenObject, use personal.specific_inventory_roi()

* make smaller gold button, trim ROI, and use grayscale to search

* remove unused roi

* fix circular import by reverting to Config inventory area and adding open_inventory_area variable, adjust code throughout

* simplify

* revert to template finding due to circular import with ui_manager

* grayscale template match

Co-authored-by: mgleed <[email protected]>

config/game.ini

@@ -86,7 +86,7 @@ chat_line_1=12,537,391,25
 save_and_exit=500,233,273,173
 play_btn=426,616,320,71
 difficulty_select=536,236,210,320
-gold_btn=927,491,267,100
+gold_btn=997,521,20,18
 inventory_gold=980,510,150,40
 gold_btn_stash=150,480,40,65
 vendor_gold_digits=186,509,97,16
@@ -134,6 +134,7 @@ quest_skill_btn=284,455,710,121
 left_inventory_tabs=31,62,385,28
 tab_indicator=31,82,385,14
 deposit_btn=454,365,186,43
+equipped_inventory_area=861,59,387,287
 
 [path]
 ; static pathes in format: x0,y0, x1,y1, x2,y2, ...

src/config.py

@@ -392,6 +392,14 @@ def load_data(self):
         self.ui_roi = {}
         for key in self.configs["game"]["parser"]["ui_roi"]:
             self.ui_roi[key] = np.array([int(x) for x in self._select_val("ui_roi", key).split(",")])
+        open_width = int(self.ui_pos["slot_width"] * self.char["num_loot_columns"])
+        # calc roi for restricted inventory area
+        self.ui_roi["restricted_inventory_area"] = self.ui_roi["right_inventory"].copy()
+        self.ui_roi["restricted_inventory_area"][0] += open_width # left
+        self.ui_roi["restricted_inventory_area"][2] -= open_width # width
+        # calc roi for open inventory area
+        self.ui_roi["open_inventory_area"] = self.ui_roi["right_inventory"].copy()
+        self.ui_roi["open_inventory_area"][2] = open_width # width
 
         self.path = {}
         for key in self.configs["game"]["parser"]["path"]:

src/inventory/consumables.py

@@ -109,7 +109,7 @@ def update_tome_key_needs(img: np.ndarray = None, item_type: str = "tp") -> bool
         match = TemplateFinder().search(
             [f"{item_type.upper()}_TOME", f"{item_type.upper()}_TOME_RED"],
             img,
-            roi = personal.specific_inventory_roi("reserved"),
+            roi = Config().ui_roi["restricted_inventory_area"],
             best_match = True,
             normalize_monitor=True
             )
@@ -122,7 +122,7 @@ def update_tome_key_needs(img: np.ndarray = None, item_type: str = "tp") -> bool
             Logger.debug(f"update_tome_key_needs: could not find {item_type}")
             return False
     elif item_type.lower() in ["key"]:
-        match = TemplateFinder().search("INV_KEY", img, roi = personal.specific_inventory_roi("reserved"), normalize_monitor = True)
+        match = TemplateFinder().search("INV_KEY", img, roi = Config().ui_roi["restricted_inventory_area"], normalize_monitor = True)
         if not match.valid:
             return False
     else:

src/inventory/personal.py

@@ -256,20 +256,6 @@ def keep_item(item_box: ItemText = None, found_item: Item = None, do_logging: bo
         return True
     return False
 
-def specific_inventory_roi(desired: str = "reserved"):
-    #roi spec: left, top, W, H
-    roi = Config().ui_roi["right_inventory"].copy()
-    open_width = Config().ui_pos["slot_width"] * Config().char["num_loot_columns"]
-    if desired == "reserved":
-        roi[0]=roi[0] + open_width
-        roi[2]=roi[2] - open_width
-    elif desired == "open":
-        roi[2]=open_width
-    else:
-        Logger.error(f"set_inventory_rois: unsupported desired={desired}")
-        return None
-    return roi
-
 def open(img: np.ndarray = None) -> np.ndarray:
     img = grab() if img is None else img
     if not is_visible(ScreenObjects.RightPanel, img):
@@ -317,8 +303,8 @@ def inspect_items(inp_img: np.ndarray = None, close_window: bool = True, game_st
             # determine the item's ROI in inventory
             cnt=0
             while True:
-                pre = mask_by_roi(img, specific_inventory_roi("open"))
-                post = mask_by_roi(hovered_item, specific_inventory_roi("open"))
+                pre = mask_by_roi(img, Config().ui_roi["open_inventory_area"])
+                post = mask_by_roi(hovered_item, Config().ui_roi["open_inventory_area"])
                 # will sometimes have equivalent diff if mouse ends up in an inconvenient place.
                 if not np.array_equal(pre, post):
                     break
@@ -358,7 +344,7 @@ def inspect_items(inp_img: np.ndarray = None, close_window: bool = True, game_st
                     if (is_unidentified := is_visible(ScreenObjects.Unidentified, item_box.data)):
                         need_id = True
                         center_mouse()
-                        tome_state, tome_pos = common.tome_state(grab(), tome_type = "id", roi = specific_inventory_roi("reserved"))
+                        tome_state, tome_pos = common.tome_state(grab(), tome_type = "id", roi = Config().ui_roi["restricted_inventory_area"])
                     if is_unidentified and tome_state is not None and tome_state == "ok":
                         common.id_item_with_tome([x_m, y_m], tome_pos)
                         need_id = False
@@ -475,7 +461,7 @@ def transfer_items(items: list, action: str = "drop", img: np.ndarray = None) ->
             pre_transfer_img = grab()
             mouse.press(button="left")
             # wait for inventory image to update indicating successful transfer / item select
-            success = wait_for_update(pre_transfer_img, specific_inventory_roi("open"), 3)
+            success = wait_for_update(pre_transfer_img, Config().ui_roi["open_inventory_area"], 3)
             mouse.release(button="left")
             if not success:
                 Logger.warning(f"transfer_items: inventory unchanged after attempting to {action} item at position {item.pos}")

src/ui_manager.py

@@ -139,7 +139,8 @@ class ScreenObjects:
     )
     GoldBtnInventory=ScreenObject(
         ref="INVENTORY_GOLD_BTN",
-        roi="gold_btn"
+        roi="gold_btn",
+        use_grayscale=True
     )
     GoldBtnStash=ScreenObject(
         ref="INVENTORY_GOLD_BTN",

src/utils/custom_mouse.py

@@ -7,7 +7,11 @@
 import math
 import time
 from typing import Union, Tuple
-
+import screen
+from config import Config
+from utils.misc import is_in_roi
+from logger import Logger
+from template_finder import TemplateFinder
 
 def isNumeric(val):
     return isinstance(val, (float, int, np.int32, np.int64, np.float32, np.float64))
@@ -254,13 +258,34 @@ def move(x, y, absolute: bool = True, randomize: Union[int, Tuple[int, int]] = 5
         for point in human_curve.points:
             _mouse.move(point[0], point[1], duration=delta)
 
+    @staticmethod
+    def _is_clicking_safe():
+        # Because of reports that botty lost equiped items, let's check if the inventory is open, and if it is, restrict the mouse move
+        mouse_pos = screen.convert_monitor_to_screen(_mouse.get_position())
+        is_inventory_open = TemplateFinder().search(
+            "INVENTORY_GOLD_BTN",
+            screen.grab(),
+            threshold=0.8,
+            roi=Config().ui_roi["gold_btn"],
+            use_grayscale=True
+        ).valid
+        if is_inventory_open:
+            is_in_equipped_area = is_in_roi(Config().ui_roi["equipped_inventory_area"], mouse_pos)
+            is_in_restricted_inventory_area = is_in_roi(Config().ui_roi["restricted_inventory_area"], mouse_pos)
+            if is_in_restricted_inventory_area or is_in_equipped_area:
+                Logger.error("Mouse wants to click in equipped area. Cancel action.")
+                return False
+        return True
+
     @staticmethod
     def click(button):
-        _mouse.click(button)
+        if button != "left" or mouse._is_clicking_safe():
+            _mouse.click(button)
 
     @staticmethod
     def press(button):
-        _mouse.press(button)
+        if button != "left" or mouse._is_clicking_safe():
+            _mouse.press(button)
 
     @staticmethod
     def release(button):
@@ -274,9 +299,21 @@ def get_position():
     def wheel(delta):
         _mouse.wheel(delta)
 
+
 if __name__ == "__main__":
-    mouse.move(100, 100)
-    time.sleep(2)
-    mouse.move(200, 200)
-    time.sleep(2)
-    mouse.move(1800, 800)
+    import os
+    import keyboard
+    keyboard.add_hotkey('f12', lambda: os._exit(1))
+    keyboard.wait("f11")
+    screen.find_and_set_window_position()
+    move_to_ok = screen.convert_screen_to_monitor((400, 420))
+    move_to_bad_equiped = screen.convert_screen_to_monitor((900, 170))
+    move_to_bad_inventory = screen.convert_screen_to_monitor((1200, 400))
+    mouse.move(*move_to_ok)
+    mouse.click("left")
+    time.sleep(1)
+    mouse.move(*move_to_bad_equiped)
+    mouse.click("left")
+    time.sleep(1)
+    mouse.move(*move_to_bad_inventory)
+    mouse.click("left")