feat. Finishing sound & agent loop fixes

src/agent.py

@@ -126,16 +126,13 @@ def build_initial_action_prompt(
         "Explicitly watch for #32770 dialogs, Explorer open/save pickers, browser download/upload flows, taskbar/start menu focus traps, context menus, disabled controls, and permission prompts.\n"
         "If focus or the foreground app may be wrong, call get_active_window, find_window, focus_window, or a focus wait helper.\n"
         "If an unexpected modal appears, pause the prior plan and resolve the modal first.\n"
+        "Do not invent new subgoals. Prefer non-visual verification when available.\n"
+        "Use wait_for_focus_change when focus transfer is expected but not yet confirmed.\n"
         "When a fresh focus check or a clear retained visual already proves the target editor or field is ready, act directly; do not re-capture the screen just to reconfirm an obvious large input area.\n"
         "You may use more than one tool in one step when that improves certainty, such as get_active_window plus detect_dialog, see_screen then enhance, or click then see_screen.\n"
         "When done, do a fresh verification pass with see_screen and add enhance if the proof is small or text-heavy.\n"
         "Then call task_complete(return=..., data={\"observed_result\": ...}).\n"
         "Include useful structured output in data.",
-        CORE_OPERATING_DOCTRINE,
-        WINDOWS_ENVIRONMENT_RULES,
-        BROWSER_WORKFLOW_RULES,
-        DIALOG_HANDLING_RULES,
-        COMPLETION_VERIFICATION_RULES,
     )
 
 
@@ -149,10 +146,6 @@ def build_no_tool_prompt(prohibited_key_combos: list[str] | tuple[str, ...] | se
         "Do not assume execute_command launches changed the foreground window; verify focus before typing.\n"
         "If a modal, picker, or browser download/upload surface is likely, resolve that first.\n"
         "Before task_complete, do a fresh verification pass with see_screen, add enhance if needed, and include data.observed_result.",
-        CORE_OPERATING_DOCTRINE,
-        WINDOWS_ENVIRONMENT_RULES,
-        DIALOG_HANDLING_RULES,
-        COMPLETION_VERIFICATION_RULES,
     )
 
 
@@ -190,11 +183,6 @@ def build_context_compaction_prompt(
         "If a fresh focus check or retained visual already proves a text field or editor is ready, act without demanding another screenshot.\n"
         "Treat execute_command app or URL launches as background until focus is explicitly verified.\n"
         "Use tools only. Finish only after a fresh verification pass with see_screen and explicit data.observed_result in task_complete.",
-        CORE_OPERATING_DOCTRINE,
-        WINDOWS_ENVIRONMENT_RULES,
-        BROWSER_WORKFLOW_RULES,
-        DIALOG_HANDLING_RULES,
-        COMPLETION_VERIFICATION_RULES,
     )
 
 
@@ -216,11 +204,6 @@ def build_blocked_action_prompt(
         "If secure desktop or UAC is suspected, stop blind retries and report the blocked state explicitly.\n"
         "Switch strategy after the fresh classification: native control instead of pixels, keyboard instead of mouse, mouse instead of keyboard, commands instead of UI, UI instead of commands, or finish if the job is already done.\n"
         f"Confirm what changed before choosing a new action. Use classify -> choose control channel -> execute one meaningful transition -> verify.{extra}",
-        CORE_OPERATING_DOCTRINE,
-        WINDOWS_ENVIRONMENT_RULES,
-        BROWSER_WORKFLOW_RULES,
-        DIALOG_HANDLING_RULES,
-        COMPLETION_VERIFICATION_RULES,
     )
 
 
@@ -235,14 +218,11 @@ def build_observation_loop_prompt(
         f"{summary_text}\n"
         "Do not keep calling broad observation tools like see_screen or get_active_window on the same unchanged state.\n"
         "Change method now: use a native window/dialog/element tool for this surface, interact with the visible control, resolve the modal, or finish if you already have proof.\n"
+        "If the requested navigation or status check already happened, do one final verification pass and then call task_complete instead of waiting or re-clicking.\n"
         f"{_prohibited_key_combo_prompt(prohibited_key_combos)}"
         "Use enhance only if a small or text-heavy control must be read before acting.\n"
         "If this window is a #32770 dialog or picker, act on the dialog instead of re-scanning the whole screen.\n"
         "Use classify -> choose control channel -> execute one meaningful transition -> verify.",
-        CORE_OPERATING_DOCTRINE,
-        WINDOWS_ENVIRONMENT_RULES,
-        DIALOG_HANDLING_RULES,
-        COMPLETION_VERIFICATION_RULES,
     )
 
 
@@ -260,7 +240,6 @@ def build_finish_likely_prompt(
             f"{_prohibited_key_combo_prompt(prohibited_key_combos)}"
             "Do not reopen menus, repeat save/export/download actions, or re-search the filesystem unless a new contradiction appears.\n"
             "Call task_complete now with a concise return string and explicit data.observed_result from the latest verification.",
-            COMPLETION_VERIFICATION_RULES,
         )
     return _compose_prompt(
         "Runtime completion evidence indicates the objective is likely already satisfied.\n"
@@ -268,7 +247,6 @@ def build_finish_likely_prompt(
         "Do one fresh verification pass now: call see_screen, add enhance only if the proof is small or text-heavy, then call task_complete.\n"
         f"{_prohibited_key_combo_prompt(prohibited_key_combos)}"
         "Do not reopen menus, repeat save/export/download actions, or re-search the filesystem unless a new contradiction appears.",
-        COMPLETION_VERIFICATION_RULES,
     )
 
 
@@ -332,10 +310,25 @@ OBSERVATION_TOOL_NAMES = VISUAL_TOOL_NAMES | WINDOW_TOOL_NAMES | DIALOG_TOOL_NAM
     "clipboard_get",
     "get_cursor_position",
 }
+OBSERVATION_NON_PROGRESS_TOOL_NAMES = VISUAL_TOOL_NAMES | {
+    "list_windows",
+    "find_window",
+    "wait_for_window",
+    "wait_for_focus_change",
+    "get_active_window",
+    "detect_dialog",
+    "wait_for_dialog_close",
+    "list_ui_elements",
+    "wait_for_ui_element",
+    "clipboard_get",
+    "get_cursor_position",
+    "sleep",
+}
 WINDOWS_ONLY_TOOL_NAMES = WINDOW_TOOL_NAMES | DIALOG_TOOL_NAMES | UI_ELEMENT_TOOL_NAMES
 MAX_ACTION_SIGNATURE_ATTEMPTS = 3
 MAX_STABLE_OBSERVATION_STEPS = 3
 FINISH_LIKELY_OBSERVATION_TOOLS = {"see_screen", "enhance", "get_active_window", "detect_dialog"}
+BROAD_REOBSERVATION_TOOL_NAMES = {"see_screen", "get_active_window", "detect_dialog", "list_windows", "sleep"}
 
 
 def normalize_disabled_tools(tool_names: set[str] | list[str] | tuple[str, ...] | None) -> list[str]:
@@ -1896,42 +1889,74 @@ class ScreenJobAgent:
         self,
         tool_names: list[str],
         active_window: dict[str, Any] | None,
+        visual_signature: str | None = None,
     ) -> None:
         if not tool_names:
             return
+        contains_action = any(name not in OBSERVATION_NON_PROGRESS_TOOL_NAMES for name in tool_names)
+        window_for_history = active_window
+        if not isinstance(window_for_history, dict) or not bool(window_for_history.get("available")):
+            fallback_window = self.last_observed_window if isinstance(self.last_observed_window, dict) else None
+            if isinstance(fallback_window, dict) and bool(fallback_window.get("available")):
+                window_for_history = fallback_window
         self.step_history.append(
             {
                 "step": self.step,
                 "tool_names": list(tool_names),
-                "window_signature": self._window_signature(active_window),
+                "window_signature": self._window_signature(window_for_history),
-                "window_summary": self._window_summary(active_window) if active_window else "",
+                "window_summary": self._window_summary(window_for_history) if window_for_history else "",
                 "had_visual": any(name in VISUAL_TOOL_NAMES for name in tool_names),
+                "contains_action": contains_action,
+                "visual_signature": str(visual_signature or "").strip(),
             }
         )
         self.step_history = self.step_history[-12:]
 
     def _stable_observation_loop(self) -> dict[str, Any] | None:
-        recent = self.step_history[-MAX_STABLE_OBSERVATION_STEPS:]
+        recent = list(reversed(self.step_history))
         if len(recent) < MAX_STABLE_OBSERVATION_STEPS:
             return None
-        if any(not entry.get("tool_names") for entry in recent):
+
-            return None
+        window_signature = ""
-        if any(not set(entry["tool_names"]).issubset(OBSERVATION_TOOL_NAMES) for entry in recent):
+        repeated_steps = 0
-            return None
+        visual_signatures: set[str] = set()
-        if any(not bool(entry.get("had_visual")) for entry in recent):
+
-            return None
+        for entry in recent:
-        signatures = {str(entry.get("window_signature") or "").strip() for entry in recent}
+            tool_names = list(entry.get("tool_names") or [])
-        signatures.discard("")
+            if not tool_names:
-        if len(signatures) != 1:
+                break
+
+            entry_signature = str(entry.get("window_signature") or "").strip()
+            if not entry_signature:
+                break
+            if not window_signature:
+                window_signature = entry_signature
+            elif entry_signature != window_signature:
+                break
+
+            if bool(entry.get("contains_action")):
+                break
+
+            if any(name in OBSERVATION_TOOL_NAMES for name in tool_names):
+                repeated_steps += 1
+                visual_signature = str(entry.get("visual_signature") or "").strip()
+                if visual_signature:
+                    visual_signatures.add(visual_signature)
+
+            if repeated_steps >= MAX_STABLE_OBSERVATION_STEPS:
+                break
+
+        if repeated_steps < MAX_STABLE_OBSERVATION_STEPS or not window_signature:
             return None
         return {
-            "signature": next(iter(signatures)),
+            "signature": window_signature,
-            "window_summary": str(recent[-1].get("window_summary") or "").strip(),
+            "window_summary": str(recent[0].get("window_summary") or "").strip(),
-            "repeated_steps": len(recent),
+            "repeated_steps": repeated_steps,
+            "visual_signature_count": len(visual_signatures),
         }
 
     def _check_observation_loop_gate(self, tool_name: str) -> dict[str, Any] | None:
-        if tool_name not in {"see_screen", "get_active_window"}:
+        if tool_name not in BROAD_REOBSERVATION_TOOL_NAMES:
             return None
         stable_loop = self._stable_observation_loop()
         if stable_loop is None:
@@ -4214,6 +4239,7 @@ class ScreenJobAgent:
             next_input: list[dict[str, Any]] = []
             step_tool_names: list[str] = []
             step_active_window: dict[str, Any] | None = None
+            step_visual_signature: str | None = None
             for tool_call in tool_calls:
                 if self._is_cancelled():
                     cancelled = True
@@ -4246,6 +4272,11 @@ class ScreenJobAgent:
                 self._emit("tool_result", {"step": self.step, "tool": name, "result": result})
                 if name == "get_active_window" and bool(result.get("ok")) and isinstance(result.get("window"), dict):
                     step_active_window = dict(result["window"])
+                if name in VISUAL_TOOL_NAMES and bool(result.get("ok")):
+                    meta = result.get("meta") if isinstance(result.get("meta"), dict) else {}
+                    visual_signature = str(meta.get("visual_signature") or "").strip()
+                    if visual_signature:
+                        step_visual_signature = visual_signature
                 next_input.append(
                     {
                         "type": "function_call_output",
@@ -4306,7 +4337,7 @@ class ScreenJobAgent:
 
             if cancelled:
                 break
-            self._record_step_history(step_tool_names, step_active_window)
+            self._record_step_history(step_tool_names, step_active_window, step_visual_signature)
             if bool(self.finish_likely_state.get("active")):
                 next_input.append(
                     {

src/cli.py

@@ -7,6 +7,7 @@ from pathlib import Path
 
 from .agent import normalize_disabled_tools
 from .config import load_app_config
+from .desktop_overlay import get_desktop_overlay_manager
 from .models import RuntimeOptions
 from .runtime import create_openai_client, run_job
 from .safety import assess_task_safety
@@ -175,6 +176,15 @@ def main(argv: list[str] | None = None) -> int:
         )
         return 1
 
+    if result.completed:
+        get_desktop_overlay_manager().show_completion(
+            job_id=artifacts.run_id,
+            objective=args.job,
+            return_message=result.return_message,
+            steps=result.steps,
+            elapsed_seconds=max(0.0, float(result.ended_at - result.started_at)),
+        )
+
     payload = {
         "completed": result.completed,
         "result": result.return_message,

src/desktop_overlay.py

@@ -7,6 +7,11 @@ import threading
 from dataclasses import dataclass
 from typing import Any
 
+try:
+    import winsound
+except Exception:  # noqa: BLE001
+    winsound = None
+
 
 @dataclass(frozen=True)
 class CompletionOverlayPayload:
@@ -28,6 +33,14 @@ class DesktopOverlayManager:
         self._warned = False
         self._auto_dismiss_ms = max(0, int(round(float(auto_dismiss_seconds) * 1000)))
 
+    def _play_completion_sound(self) -> None:
+        if os.name != "nt" or winsound is None:
+            return
+        try:
+            winsound.MessageBeep(winsound.MB_ICONASTERISK)
+        except Exception as exc:  # noqa: BLE001
+            self.logger.debug("Completion sound failed (%s: %s)", type(exc).__name__, exc)
+
     def show_completion(
         self,
         *,
@@ -37,6 +50,7 @@ class DesktopOverlayManager:
         steps: int,
         elapsed_seconds: float,
     ) -> None:
+        self._play_completion_sound()
         if os.name != "nt":
             self._disable_once("Desktop completion HUD is only enabled on Windows.")
             return

tests/test_agent_tools.py

@@ -495,6 +495,128 @@ def test_observation_loop_blocks_repeated_broad_reobservation(tmp_path: Path, mo
     assert blocked["window_summary"] == "Save as [#32770]"
 
 
+def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypatch) -> None:
+    agent = _build_agent(tmp_path, monkeypatch)
+    agent.step_history = [
+        {
+            "step": 40,
+            "tool_names": ["click"],
+            "window_signature": "123|#32770|Save as",
+            "window_summary": "Save as [#32770]",
+            "had_visual": False,
+            "contains_action": True,
+        },
+        {
+            "step": 41,
+            "tool_names": ["see_screen"],
+            "window_signature": "123|#32770|Save as",
+            "window_summary": "Save as [#32770]",
+            "had_visual": True,
+            "contains_action": False,
+        },
+        {
+            "step": 42,
+            "tool_names": ["sleep"],
+            "window_signature": "123|#32770|Save as",
+            "window_summary": "Save as [#32770]",
+            "had_visual": False,
+            "contains_action": False,
+        },
+        {
+            "step": 43,
+            "tool_names": ["get_active_window"],
+            "window_signature": "123|#32770|Save as",
+            "window_summary": "Save as [#32770]",
+            "had_visual": False,
+            "contains_action": False,
+        },
+        {
+            "step": 44,
+            "tool_names": ["sleep"],
+            "window_signature": "123|#32770|Save as",
+            "window_summary": "Save as [#32770]",
+            "had_visual": False,
+            "contains_action": False,
+        },
+        {
+            "step": 45,
+            "tool_names": ["see_screen"],
+            "window_signature": "123|#32770|Save as",
+            "window_summary": "Save as [#32770]",
+            "had_visual": True,
+            "contains_action": False,
+        },
+    ]
+
+    blocked = agent._dispatch_tool("see_screen", {})
+
+    assert blocked["ok"] is False
+    assert blocked["blocked"] is True
+    assert blocked["blocked_reason"] == "observation_loop"
+    assert blocked["repeated_steps"] == 3
+
+
+def test_observation_loop_treats_focus_window_as_action_progress(tmp_path: Path, monkeypatch) -> None:
+    agent = _build_agent(tmp_path, monkeypatch)
+
+    agent._record_step_history(["focus_window"], {"hwnd": 123, "class_name": "#32770", "title": "Save as"})
+
+    entry = agent.step_history[-1]
+    assert entry["contains_action"] is True
+
+
+def test_observation_loop_requires_non_empty_window_signature(tmp_path: Path, monkeypatch) -> None:
+    agent = _build_agent(tmp_path, monkeypatch)
+    agent.step_history = [
+        {
+            "step": 50,
+            "tool_names": ["see_screen"],
+            "window_signature": "",
+            "window_summary": "",
+            "had_visual": True,
+            "contains_action": False,
+        },
+        {
+            "step": 51,
+            "tool_names": ["get_active_window"],
+            "window_signature": "",
+            "window_summary": "",
+            "had_visual": False,
+            "contains_action": False,
+        },
+        {
+            "step": 52,
+            "tool_names": ["see_screen"],
+            "window_signature": "",
+            "window_summary": "",
+            "had_visual": True,
+            "contains_action": False,
+        },
+    ]
+
+    assert agent._stable_observation_loop() is None
+
+
+def test_record_step_history_reuses_last_observed_window_for_visual_only_steps(tmp_path: Path, monkeypatch) -> None:
+    agent = _build_agent(tmp_path, monkeypatch)
+    agent.last_observed_window = {
+        "available": True,
+        "hwnd": 321,
+        "class_name": "ApplicationFrameWindow",
+        "title": "Settings",
+    }
+
+    agent._record_step_history(["see_screen"], None, "sig-a")
+    agent._record_step_history(["get_active_window"], None)
+    agent._record_step_history(["detect_dialog"], None)
+
+    stable = agent._stable_observation_loop()
+
+    assert stable is not None
+    assert stable["window_summary"] == "Settings [ApplicationFrameWindow]"
+    assert stable["repeated_steps"] == 3
+
+
 def test_repeated_ambiguous_action_requires_verification_and_then_blocks(tmp_path: Path, monkeypatch) -> None:
     agent = _build_agent(tmp_path, monkeypatch)
     type_args = {"text": "repeat me"}

tests/test_cli.py

@@ -9,6 +9,14 @@ from src.config import AppConfig
 from src.models import AgentResult, RunArtifacts, UsageSummary
 
 
+class _OverlayRecorder:
+    def __init__(self) -> None:
+        self.calls: list[dict[str, Any]] = []
+
+    def show_completion(self, **kwargs: Any) -> None:
+        self.calls.append(kwargs)
+
+
 def test_cli_emits_structured_return_and_data(monkeypatch: Any, capsys, tmp_path: Path) -> None:
     config = AppConfig(
         openai_api_key="test_key",
@@ -56,16 +64,28 @@ def test_cli_emits_structured_return_and_data(monkeypatch: Any, capsys, tmp_path
         )
         return result, artifacts
 
+    overlay = _OverlayRecorder()
+
     monkeypatch.setattr(cli_module, "load_app_config", fake_load_app_config)
     monkeypatch.setattr(cli_module, "assess_task_safety", fake_assess_task_safety)
     monkeypatch.setattr(cli_module, "run_job", fake_run_job)
     monkeypatch.setattr(cli_module, "create_openai_client", lambda *_args, **_kwargs: object())
+    monkeypatch.setattr(cli_module, "get_desktop_overlay_manager", lambda: overlay)
 
     code = cli_module.main(["Open amazon.de"])
     assert code == 0
 
     out = capsys.readouterr().out
     payload = json.loads(out)
+    assert overlay.calls == [
+        {
+            "job_id": "20260527_000001",
+            "objective": "Open amazon.de",
+            "return_message": "Task completed successfully",
+            "steps": 3,
+            "elapsed_seconds": 2.5,
+        }
+    ]
     assert payload["response"]["return"] == "Task completed successfully"
     assert payload["response"]["data"] == "file1.txt\nfile2.txt"
     assert payload["return"] == "Task completed successfully"

tests/test_desktop_overlay.py

@@ -4,6 +4,7 @@ import types
 from collections import deque
 from typing import Any
 
+import src.desktop_overlay as desktop_overlay_module
 from src.desktop_overlay import CompletionOverlayPayload, DesktopOverlayManager
 
 
@@ -127,6 +128,25 @@ class _FakeTkModule(types.SimpleNamespace):
         return _FakeButton(root._root, command=command)
 
 
+def test_show_completion_plays_sound_even_without_overlay_thread(monkeypatch: Any) -> None:
+    manager = DesktopOverlayManager()
+    calls: list[str] = []
+
+    monkeypatch.setattr(desktop_overlay_module.os, "name", "nt", raising=False)
+    monkeypatch.setattr(manager, "_play_completion_sound", lambda: calls.append("sound"))
+    monkeypatch.setattr(manager, "_ensure_thread", lambda: False)
+
+    manager.show_completion(
+        job_id="job-123",
+        objective="Write a report",
+        return_message="Finished",
+        steps=5,
+        elapsed_seconds=12.4,
+    )
+
+    assert calls == ["sound"]
+
+
 def test_completion_overlay_auto_dismisses(monkeypatch: Any) -> None:
     root = _FakeTk()
     fake_tk = _FakeTkModule(root)
@@ -147,3 +167,15 @@ def test_completion_overlay_auto_dismisses(monkeypatch: Any) -> None:
 
     assert any(delay == 10 for delay in root.scheduled_delays)
     assert root.cards[0]._exists is False
+
+
+def test_play_completion_sound_uses_winsound_message_beep(monkeypatch: Any) -> None:
+    calls: list[int] = []
+    fake_winsound = types.SimpleNamespace(MB_ICONASTERISK=64, MessageBeep=lambda value: calls.append(value))
+
+    monkeypatch.setattr(desktop_overlay_module.os, "name", "nt", raising=False)
+    monkeypatch.setattr(desktop_overlay_module, "winsound", fake_winsound)
+
+    DesktopOverlayManager()._play_completion_sound()
+
+    assert calls == [64]