Add pytesseract OCR, click_text interact action, and interact verify endpoint

tests/test_ocr_and_interact.py

@@ -0,0 +1,139 @@
+import sys
+
+from PIL import Image
+from fastapi.testclient import TestClient
+
+from server import services
+from server.app import app
+from server.models import ClickTextAction
+
+
+def test_extract_ocr_items_normalization(monkeypatch):
+    class FakeOutput:
+        DICT = "DICT"
+
+    class FakeTesseract:
+        Output = FakeOutput
+
+        @staticmethod
+        def image_to_data(_image, lang, config, output_type):
+            assert lang == "eng"
+            assert output_type == "DICT"
+            return {
+                "text": ["hello", " ", "world"],
+                "conf": ["95.0", "-1", "62.5"],
+                "left": [10, 12, 40],
+                "top": [20, 25, 60],
+                "width": [30, 10, 50],
+                "height": [10, 10, 12],
+            }
+
+    monkeypatch.setitem(sys.modules, "pytesseract", FakeTesseract)
+    items = services.extract_ocr_items(Image.new("RGB", (100, 100)), origin_x=100, origin_y=200, min_confidence=60, lang="eng", psm=None)
+    assert len(items) == 2
+    assert items[0]["text"] == "hello"
+    assert items[0]["bbox"]["x"] == 110
+    assert items[0]["center"]["y"] == 225
+    assert items[1]["text"] == "world"
+
+
+def test_resolve_text_match_contains_exact_regex_and_nth():
+    items = [
+        {"text": "Save", "confidence": 70},
+        {"text": "Save as", "confidence": 96},
+        {"text": "SAVE", "confidence": 88},
+    ]
+    contains = services._resolve_text_match(ClickTextAction(text="save", match="contains", occurrence="first"), items)
+    assert contains["text"] == "Save"
+    best = services._resolve_text_match(ClickTextAction(text="save", match="contains", occurrence="best"), items)
+    assert best["text"] == "Save as"
+    exact_case = services._resolve_text_match(
+        ClickTextAction(text="SAVE", match="exact", case_sensitive=True, occurrence="first"),
+        items,
+    )
+    assert exact_case["text"] == "SAVE"
+    regex_nth = services._resolve_text_match(ClickTextAction(text="^Save", match="regex", occurrence="nth", nth=2), items)
+    assert regex_nth["text"] == "Save as"
+
+
+def test_interact_click_text_region_optional(monkeypatch):
+    monkeypatch.setattr(services, "select_display", lambda screen: ({"screen": screen}, [], {"requested": screen, "selected": screen, "fallback": False}))
+    monkeypatch.setattr(
+        services,
+        "capture_region_image",
+        lambda screen, x, y, w, h: (Image.new("RGB", (20, 20)), {"x": x or 0, "y": y or 0, "width": w or 20, "height": h or 20}, {}, [], {}),
+    )
+    monkeypatch.setattr(
+        services,
+        "extract_ocr_items",
+        lambda *args, **kwargs: [
+            {
+                "text": "Apply",
+                "confidence": 93.0,
+                "bbox": {"x": 10, "y": 20, "width": 20, "height": 10},
+                "center": {"x": 20, "y": 25},
+                "region_relative_bbox": {"x": 10, "y": 20, "width": 20, "height": 10},
+            }
+        ],
+    )
+
+    client = TestClient(app)
+    response = client.post(
+        "/interact",
+        json={"screen": 0, "action": {"action": "click_text", "dry_run": True, "click_text": {"text": "Apply", "match": "contains"}}},
+    )
+    assert response.status_code == 200
+    body = response.json()["data"]
+    assert body["resolved_target"]["x"] == 20
+    assert body["click_text_match"]["matched"]["text"] == "Apply"
+
+
+def test_see_ocr_off_on_contract(monkeypatch):
+    monkeypatch.setattr(
+        "server.app.capture_region_image",
+        lambda *args, **kwargs: (Image.new("RGB", (10, 10)), {"x": 0, "y": 0, "width": 10, "height": 10}, {"screen": 0}, [], {}),
+    )
+    monkeypatch.setattr("server.app.encode_image", lambda *args, **kwargs: "abc")
+    monkeypatch.setattr("server.app.extract_ocr_items", lambda *args, **kwargs: [{"text": "x"}])
+
+    client = TestClient(app)
+    off = client.post("/see", json={"ocr": False, "with_grid": False})
+    assert off.status_code == 200
+    assert "ocr" not in off.json()["data"]["meta"]
+    on = client.post("/see", json={"ocr": True, "with_grid": False})
+    assert on.status_code == 200
+    assert on.json()["data"]["meta"]["ocr"][0]["text"] == "x"
+
+
+def test_interact_verify_success_and_timeout(monkeypatch):
+    calls = {"n": 0}
+    monkeypatch.setattr(services, "exec_action", lambda action, screen=0: {"action": action.action, "executed": True})
+
+    def fake_verify(_spec):
+        calls["n"] += 1
+        return {"ok": calls["n"] >= 2, "matches": [], "items_count": 1, "screen": {"selected": 0}, "region": {"x": 0, "y": 0, "width": 1, "height": 1}}
+
+    monkeypatch.setattr(services, "_verify_ocr_text_near_point", fake_verify)
+    client = TestClient(app)
+    payload = {
+        "action": {"screen": 0, "action": {"action": "type", "text": "hello"}},
+        "verify": {"type": "ocr_text_near_point", "text": "hello", "x": 100, "y": 100},
+        "check_interval_ms": 10,
+        "timeout_ms": 500,
+    }
+    ok_resp = client.post("/interact/verify", json=payload)
+    assert ok_resp.status_code == 200
+    ok_data = ok_resp.json()["data"]
+    assert ok_data["verified"] is True
+    assert ok_data["attempts"] == 2
+
+    monkeypatch.setattr(
+        services,
+        "_verify_ocr_text_near_point",
+        lambda _spec: {"ok": False, "matches": [], "items_count": 0, "screen": {"selected": 0}, "region": {"x": 0, "y": 0, "width": 1, "height": 1}},
+    )
+    timeout_resp = client.post("/interact/verify", json=payload)
+    assert timeout_resp.status_code == 200
+    timeout_data = timeout_resp.json()["data"]
+    assert timeout_data["verified"] is False
+    assert timeout_data["attempts"] >= 1