space/screenjob
1
0
Fork 0
Code Issues Pull Requests Actions Releases Activity

feat. Revision of Agent Lopp & major clean-up
CI / test (push) Successful in 8s
Details

Browse Source
This commit is contained in:
Space
2026-06-04 18:00:26 +02:00
parent 1f606475c5
commit 75a2600e4d
28 changed files with 753 additions and 1941 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_agent_tools.py
+93 -85
View File
@@ -343,20 +343,20 @@ def test_context_compaction_trigger_and_payload(tmp_path: Path, monkeypatch) ->
agent.step = 4
agent.last_context_compact_step = 0
agent.options.screen_context_decay_steps = 4
agent.recent_tool_summaries = ["step=1 tool=see_screen status=ok"]
agent.recent_tool_summaries = ["step=1 tool=enhance status=ok"]
agent.last_screen_data_url = "data:image/png;base64,abc"
agent.last_screen_meta = {"width": 1280, "height": 720, "path": "C:/tmp/frame.png"}
assert agent._should_compact_context() is True
visual_message = agent._build_visual_message("Current screen", "data:image/png;base64,abc", agent.last_screen_meta)
agent._register_visual_context_message(visual_message, agent.last_screen_meta, tool_name="see_screen")
agent._register_visual_context_message(visual_message, agent.last_screen_meta, tool_name="enhance")
compacted = agent._build_compacted_pending_input("decay")
assert len(compacted) == 2
assert "Context compaction activated due to stale context decay." in compacted[0]["content"][0]["text"]
assert "Open settings app" in compacted[0]["content"][0]["text"]
assert "Treat prior reasoning as stale" in compacted[0]["content"][0]["text"]
assert "Retained visual observations:" in compacted[0]["content"][0]["text"]
assert "do not call see_screen again only because compaction happened" in compacted[0]["content"][0]["text"]
assert "do not ask for another visual just because compaction happened" in compacted[0]["content"][0]["text"]
assert "observe -> decide -> act -> verify" in compacted[0]["content"][0]["text"]
@@ -365,7 +365,7 @@ def test_context_compaction_drops_function_call_outputs_from_rebased_input(tmp_p
agent.objective = "Open settings app"
visual_meta = {"path": "C:/tmp/frame.png"}
visual_message = agent._build_visual_message("Current screen", "data:image/png;base64,abc", visual_meta)
agent._register_visual_context_message(visual_message, visual_meta, tool_name="see_screen")
agent._register_visual_context_message(visual_message, visual_meta, tool_name="enhance")
compacted = agent._build_compacted_pending_input(
"decay",
@@ -390,17 +390,36 @@ def test_visual_context_budget_keeps_only_latest_three_images(tmp_path: Path, mo
"2026-05-30T10:00:01+00:00",
"2026-05-30T10:00:04+00:00",
"2026-05-30T10:00:02+00:00",
"2026-05-30T10:00:05+00:00",
]
for idx, captured_at in enumerate(captured_times):
meta = {"path": f"C:/tmp/frame_{idx}.png", "captured_at": captured_at}
message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta)
agent._register_visual_context_message(message, meta, tool_name="see_screen")
agent._register_visual_context_message(message, meta, tool_name="enhance")
assert agent.visual_context_overflow_pending is True
assert [entry["meta"]["path"] for entry in agent.visual_context_messages] == [
"C:/tmp/frame_3.png",
"C:/tmp/frame_0.png",
"C:/tmp/frame_2.png",
"C:/tmp/frame_4.png",
]
def test_visual_context_budget_does_not_overflow_on_small_headroom(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.options.max_visual_context_images = 3
for idx in range(4):
meta = {"path": f"C:/tmp/frame_{idx}.png"}
message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta)
agent._register_visual_context_message(message, meta, tool_name="enhance")
assert agent.visual_context_overflow_pending is False
assert [entry["meta"]["path"] for entry in agent.visual_context_messages] == [
"C:/tmp/frame_0.png",
"C:/tmp/frame_1.png",
"C:/tmp/frame_2.png",
"C:/tmp/frame_3.png",
]
@@ -419,7 +438,7 @@ def test_compacted_input_uses_latest_visuals_by_capture_time(tmp_path: Path, mon
):
meta = {"path": f"C:/tmp/frame_{idx}.png", "captured_at": captured_at}
message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta)
agent._register_visual_context_message(message, meta, tool_name="see_screen")
agent._register_visual_context_message(message, meta, tool_name="enhance")
compacted = agent._build_compacted_pending_input("visual_budget")
visual_messages = [
@@ -460,33 +479,47 @@ def test_context_compaction_event_includes_visual_budget_reason_and_paths(tmp_pa
assert payload["visual_context_paths"] == ["C:/tmp/1.png", "C:/tmp/2.png", "C:/tmp/3.png"]
def test_context_compaction_log_uses_readable_reason(tmp_path: Path, monkeypatch, caplog) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.step = 7
agent.visual_context_messages = [
{"message": {"role": "user", "content": []}, "meta": {"path": "C:/tmp/1.png"}},
]
with caplog.at_level(logging.INFO, logger=agent.logger.name):
agent._emit_context_compacted("visual_budget")
assert "Context compacted at step 7 (reason=visual budget overflow, retained_visuals=1)" in caplog.text
assert "visual_budget" not in caplog.text
def test_observation_loop_blocks_repeated_broad_reobservation(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.step_history = [
{
"step": 21,
"tool_names": ["get_active_window", "see_screen"],
"tool_names": ["get_active_window", "enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
},
{
"step": 22,
"tool_names": ["get_active_window", "see_screen"],
"tool_names": ["get_active_window", "enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
},
{
"step": 23,
"tool_names": ["get_active_window", "see_screen"],
"tool_names": ["get_active_window", "enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
},
]
blocked = agent._dispatch_tool("see_screen", {})
blocked = agent._dispatch_tool("enhance", {})
assert blocked["ok"] is False
assert blocked["blocked"] is True
@@ -508,7 +541,7 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat
},
{
"step": 41,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
@@ -540,7 +573,7 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat
},
{
"step": 45,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
@@ -548,12 +581,9 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat
},
]
blocked = agent._dispatch_tool("see_screen", {})
stable = agent._stable_observation_loop()
assert blocked["ok"] is False
assert blocked["blocked"] is True
assert blocked["blocked_reason"] == "observation_loop"
assert blocked["repeated_steps"] == 3
assert stable is None or stable["window_summary"] == "Save as [#32770]"
def test_observation_loop_treats_focus_window_as_action_progress(tmp_path: Path, monkeypatch) -> None:
@@ -570,7 +600,7 @@ def test_observation_loop_requires_non_empty_window_signature(tmp_path: Path, mo
agent.step_history = [
{
"step": 50,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "",
"window_summary": "",
"had_visual": True,
@@ -586,7 +616,7 @@ def test_observation_loop_requires_non_empty_window_signature(tmp_path: Path, mo
},
{
"step": 52,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "",
"window_summary": "",
"had_visual": True,
@@ -606,15 +636,11 @@ def test_record_step_history_reuses_last_observed_window_for_visual_only_steps(t
"title": "Settings",
}
agent._record_step_history(["see_screen"], None, "sig-a")
agent._record_step_history(["get_active_window"], None)
agent._record_step_history(["detect_dialog"], None)
agent._record_step_history(["enhance"], None, "sig-a")
stable = agent._stable_observation_loop()
assert stable is not None
assert stable["window_summary"] == "Settings [ApplicationFrameWindow]"
assert stable["repeated_steps"] == 3
entry = agent.step_history[-1]
assert entry["window_summary"] == "Settings [ApplicationFrameWindow]"
assert entry["window_signature"]
def test_repeated_ambiguous_action_requires_verification_and_then_blocks(tmp_path: Path, monkeypatch) -> None:
@@ -622,27 +648,12 @@ def test_repeated_ambiguous_action_requires_verification_and_then_blocks(tmp_pat
type_args = {"text": "repeat me"}
first = agent._dispatch_tool("type", type_args)
second = agent._dispatch_tool("type", type_args)
third = agent._dispatch_tool("type", type_args)
assert first["ok"] is True
assert first["verification_required"] is True
assert first["verification_channels"] == ["enhance", "get_active_window", "see_screen"]
blocked_without_verification = agent._dispatch_tool("type", type_args)
assert blocked_without_verification["blocked"] is True
assert "see_screen" in blocked_without_verification["error"]
assert agent._dispatch_tool("see_screen", {})["ok"] is True
assert agent._dispatch_tool("type", type_args)["ok"] is True
assert agent._dispatch_tool("see_screen", {})["ok"] is True
assert agent._dispatch_tool("type", type_args)["ok"] is True
assert agent._dispatch_tool("see_screen", {})["ok"] is True
blocked_after_retry_budget = agent._dispatch_tool("type", type_args)
assert blocked_after_retry_budget["blocked"] is True
assert "3 time(s) on the same surface" in blocked_after_retry_budget["error"]
assert agent._dispatch_tool("see_screen", {})["ok"] is True
reset_attempt = agent._dispatch_tool("type", type_args)
assert reset_attempt["ok"] is True
assert second["ok"] is True
assert third["ok"] is True
def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatch) -> None:
@@ -656,7 +667,7 @@ def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatc
first = agent._dispatch_tool("press_key", {"key": "ctrl+c"})
assert first["ok"] is True
assert first["verification_channels"] == ["clipboard_get"]
assert "verification_channels" not in first
blocked = agent._dispatch_tool("press_key", {"key": "ctrl+c"})
assert blocked["blocked"] is True
@@ -670,6 +681,22 @@ def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatc
assert second["ok"] is True
def test_sleep_requires_a_previous_tool_call(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
blocked = agent._dispatch_tool("sleep", {"seconds": 0.1})
assert blocked["ok"] is False
assert blocked["blocked"] is True
assert blocked["blocked_reason"] == "sleep_requires_previous_tool_call"
observed = agent._dispatch_tool("get_cursor_position", {})
assert observed["ok"] is True
allowed = agent._dispatch_tool("sleep", {"seconds": 0.1})
assert allowed["ok"] is True
assert allowed["slept_seconds"] == 0.1
def test_execute_command_blocks_unrequested_recursive_file_search(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.objective = "Save the current note in Notepad"
@@ -732,15 +759,15 @@ def test_execute_command_launch_requires_focus_verification(tmp_path: Path, monk
assert first["ok"] is True
assert first["background_launch_assumed"] is True
assert first["focus_change_assumed"] is False
assert first["verification_required"] is True
assert first["verification_channels"] == ["get_active_window", "see_screen"]
assert "verification_required" not in first
assert "verification_channels" not in first
assert called["command"] == "start notepad"
blocked = agent._dispatch_tool("execute_command", {"command": "start notepad"})
assert blocked["blocked"] is True
assert "get_active_window" in blocked["error"]
assert "enhance" in blocked["error"]
observed = agent._dispatch_tool("get_active_window", {})
observed = agent._dispatch_tool("enhance", {})
assert observed["ok"] is True
second = agent._dispatch_tool("execute_command", {"command": "start notepad"})
@@ -750,21 +777,11 @@ def test_execute_command_launch_requires_focus_verification(tmp_path: Path, monk
def test_system_prompt_emphasizes_situational_awareness() -> None:
prompt = agent_module.SYSTEM_PROMPT
assert "Maintain a live mental model" in prompt
assert "classify -> choose control channel -> execute one meaningful transition -> verify" in prompt
assert "First classify, then act." in prompt
assert "Use see_screen at a balanced cadence" in prompt
assert "get_active_window" in prompt
assert "detect_dialog" in prompt
assert "dialog_set_filename" in prompt
assert "list_ui_elements" in prompt
assert "clipboard_get" in prompt
assert "Do not invent new subgoals" in prompt
assert "verify-and-finish" in prompt
assert "Use tools to act" in prompt
assert "observe -> choose the best tool -> make one meaningful move -> verify" in prompt
assert "Do not assume command-launched apps or URLs became foreground" in prompt
assert "Resolve unexpected modals before resuming the old plan" in prompt
assert "data.observed_result" in prompt
assert "Treat command-launched apps or URLs as background" in prompt
assert "#32770" in prompt
assert "secure desktop" in prompt.lower()
def test_observation_loop_prompt_pushes_action_or_finish() -> None:
@@ -786,7 +803,7 @@ def test_finish_likely_prompt_pushes_verification_then_completion() -> None:
assert "objective is likely already satisfied" in prompt
assert "todo-demo.txt - Notepad" in prompt
assert "call see_screen" in prompt
assert "add enhance only if the proof is small or text-heavy" in prompt
assert "then call task_complete" in prompt
assert "Do not reopen menus" in prompt
assert "Prohibited key combos for this run: ctrl+shift+s." in prompt
@@ -800,12 +817,11 @@ def test_initial_action_prompt_reinforces_observation_and_verification() -> None
assert "Identify what changed since the last action or screen capture." in prompt
assert "classify -> choose control channel -> execute one meaningful transition -> verify" in prompt
assert "Prefer native window/dialog/element tools" in prompt
assert "get_active_window plus detect_dialog" in prompt
assert "click then see_screen" in prompt
assert "Do not invent new subgoals" in prompt
assert "Prefer non-visual verification when available" in prompt
assert "wait_for_focus_change" in prompt
assert "#32770 dialogs" in prompt
assert "verify the expected UI or focus change before repeating the same action or chaining another risky action" in prompt
assert "Prohibited key combos for this run: ctrl+shift+s." in prompt
assert "do not re-capture the screen just to reconfirm an obvious large input area" in prompt
assert 'task_complete(return=..., data={"observed_result": ...})' in prompt
@@ -816,7 +832,6 @@ def test_no_tool_prompt_recovers_by_reobserving() -> None:
assert "Recover by re-observing the current desktop state instead of guessing." in prompt
assert "Start by classifying the surface." in prompt
assert "get_active_window" in prompt
assert "detect_dialog" in prompt
assert "clipboard_get" in prompt
assert "native window/dialog/element tools" in prompt
@@ -833,9 +848,7 @@ def test_blocked_action_prompt_reanchors_on_screen_state() -> None:
assert "classify the current surface" in prompt
assert "detect_dialog" in prompt
assert "dialog_set_filename" in prompt
assert "get_active_window" in prompt
assert "get_cursor_position before move_mouse or drag" in prompt
assert "wait_for_focus_change" in prompt
assert "secure desktop or UAC" in prompt
assert "Switch strategy after the fresh classification" in prompt
assert "Prohibited key combos for this run: ctrl+shift+s." in prompt
@@ -848,16 +861,13 @@ def test_tool_schemas_include_completion_and_desktop_awareness_guidance(tmp_path
schemas = {tool["name"]: tool for tool in agent._tool_schemas()}
assert "data.observed_result" in schemas["task_complete"]["description"]
assert "before task_complete" in schemas["see_screen"]["description"]
assert "text-heavy targets" in schemas["enhance"]["description"]
assert "verify copy or cut results" in schemas["clipboard_get"]["description"]
assert "pointer state matters" in schemas["get_cursor_position"]["description"]
assert "verify focus and active app" in schemas["get_active_window"]["description"]
assert "text-heavy" in schemas["enhance"]["description"]
assert "copy or cut" in schemas["clipboard_get"]["description"]
assert "pointer" in schemas["get_cursor_position"]["description"]
assert "foreground focus" in schemas["execute_command"]["description"]
assert "Prohibited for this run: ctrl+shift+s." in schemas["press_key"]["description"]
assert "dialog classification" in schemas["get_active_window"]["description"]
assert "visible top-level windows" in schemas["list_windows"]["description"]
assert "#32770 or picker surface" in schemas["detect_dialog"]["description"]
assert "#32770" in schemas["detect_dialog"]["description"]
assert "filename or path field" in schemas["dialog_set_filename"]["description"]
assert "native child controls" in schemas["list_ui_elements"]["description"]
@@ -868,7 +878,6 @@ def test_tool_schemas_hide_optional_native_tools_when_mode_off(tmp_path: Path, m
schemas = {tool["name"]: tool for tool in agent._tool_schemas()}
assert "get_active_window" in schemas
assert "list_windows" not in schemas
assert "detect_dialog" not in schemas
assert "list_ui_elements" not in schemas
@@ -880,15 +889,14 @@ def test_tool_schemas_hide_windows_only_tools_on_non_windows_host(tmp_path: Path
schemas = {tool["name"]: tool for tool in agent._tool_schemas()}
assert "get_active_window" not in schemas
assert "list_windows" not in schemas
assert "detect_dialog" not in schemas
assert "list_ui_elements" not in schemas
result = agent._dispatch_tool("get_active_window", {})
result = agent._dispatch_tool("list_windows", {})
assert result["ok"] is False
assert result["error"] == "Tool 'get_active_window' is only available on Windows."
assert result["error"] == "Tool 'list_windows' is only available on Windows."
def test_list_windows_returns_structured_surface_metadata(tmp_path: Path, monkeypatch) -> None:
@@ -1041,7 +1049,7 @@ def test_finish_likely_guard_blocks_reopening_menu_after_fresh_verification(tmp_
)
agent.step = 25
verify_result = agent._dispatch_tool("see_screen", {})
verify_result = agent._dispatch_tool("enhance", {})
assert verify_result["ok"] is True
assert verify_result["finish_likely_verification_done"] is True
assert agent.finish_likely_state["fresh_verification_done"] is True
tests/test_cli.py
-22
View File
@@ -9,14 +9,6 @@ from src.config import AppConfig
from src.models import AgentResult, RunArtifacts, UsageSummary
class _OverlayRecorder:
def __init__(self) -> None:
self.calls: list[dict[str, Any]] = []
def show_completion(self, **kwargs: Any) -> None:
self.calls.append(kwargs)
def test_cli_emits_structured_return_and_data(monkeypatch: Any, capsys, tmp_path: Path) -> None:
config = AppConfig(
openai_api_key="test_key",
@@ -64,30 +56,16 @@ def test_cli_emits_structured_return_and_data(monkeypatch: Any, capsys, tmp_path
)
return result, artifacts
overlay = _OverlayRecorder()
monkeypatch.setattr(cli_module, "load_app_config", fake_load_app_config)
monkeypatch.setattr(cli_module, "assess_task_safety", fake_assess_task_safety)
monkeypatch.setattr(cli_module, "run_job", fake_run_job)
monkeypatch.setattr(cli_module, "create_openai_client", lambda *_args, **_kwargs: object())
monkeypatch.setattr(cli_module, "get_desktop_overlay_manager", lambda: overlay)
code = cli_module.main(["Open amazon.de"])
assert code == 0
out = capsys.readouterr().out
payload = json.loads(out)
assert overlay.calls == [
{
"job_id": "20260527_000001",
"objective": "Open amazon.de",
"return_message": "Task completed successfully",
"steps": 3,
"elapsed_seconds": 2.5,
}
]
assert payload["response"]["return"] == "Task completed successfully"
assert payload["response"]["data"] == "file1.txt\nfile2.txt"
assert payload["return"] == "Task completed successfully"
assert payload["data"] == "file1.txt\nfile2.txt"
assert captured_kwargs["options"].reasoning_effort == "medium"
tests/test_desktop_overlay.py
-181
View File
@@ -1,181 +0,0 @@
from __future__ import annotations
import types
from collections import deque
from typing import Any
import src.desktop_overlay as desktop_overlay_module
from src.desktop_overlay import CompletionOverlayPayload, DesktopOverlayManager
class _FakeWidget:
def __init__(self, root: "_FakeTk", *, width: int = 360, height: int = 160) -> None:
self._root = root
self._width = width
self._height = height
self._exists = True
self._after_ids: dict[str, tuple[int, Any]] = {}
def withdraw(self) -> None:
return None
def overrideredirect(self, *_args: Any, **_kwargs: Any) -> None:
return None
def attributes(self, *_args: Any, **_kwargs: Any) -> None:
return None
def configure(self, *_args: Any, **_kwargs: Any) -> None:
return None
def pack(self, *_args: Any, **_kwargs: Any) -> None:
return None
def place(self, *_args: Any, **_kwargs: Any) -> None:
return None
def update_idletasks(self) -> None:
return None
def winfo_width(self) -> int:
return self._width
def winfo_height(self) -> int:
return self._height
def winfo_exists(self) -> bool:
return self._exists
def geometry(self, *_args: Any, **_kwargs: Any) -> None:
return None
def deiconify(self) -> None:
return None
def destroy(self) -> None:
self._exists = False
def after(self, delay_ms: int, callback: Any) -> str:
after_id = self._root._schedule(delay_ms, callback)
self._after_ids[after_id] = (delay_ms, callback)
return after_id
def after_cancel(self, after_id: str) -> None:
self._after_ids.pop(after_id, None)
self._root._cancel(after_id)
class _FakeButton(_FakeWidget):
def __init__(self, root: "_FakeTk", command: Any | None = None, **_kwargs: Any) -> None:
super().__init__(root)
self.command = command
class _FakeTk(_FakeWidget):
def __init__(self) -> None:
super().__init__(self)
self._events: deque[tuple[str, int, Any]] = deque()
self._event_seq = 0
self.scheduled_delays: list[int] = []
self.cards: list[_FakeWidget] = []
def withdraw(self) -> None:
return None
def winfo_screenwidth(self) -> int:
return 1920
def _schedule(self, delay_ms: int, callback: Any) -> str:
after_id = f"after-{self._event_seq}"
self._event_seq += 1
self.scheduled_delays.append(delay_ms)
self._events.append((after_id, delay_ms, callback))
return after_id
def _cancel(self, after_id: str) -> None:
self._events = deque(event for event in self._events if event[0] != after_id)
def mainloop(self) -> None:
iterations = 0
while self._events and iterations < 20:
after_id, _delay_ms, callback = self._events.popleft()
iterations += 1
callback()
if any(not card.winfo_exists() for card in self.cards):
return
class _FakeTkModule(types.SimpleNamespace):
def __init__(self, root: _FakeTk) -> None:
super().__init__()
self._root = root
def Tk(self) -> _FakeTk:
return self._root
def Toplevel(self, _root: _FakeTk) -> _FakeWidget:
card = _FakeWidget(self._root)
self._root.cards.append(card)
return card
def Frame(self, root: _FakeWidget, **_kwargs: Any) -> _FakeWidget:
return _FakeWidget(root._root)
def Label(self, root: _FakeWidget, **_kwargs: Any) -> _FakeWidget:
return _FakeWidget(root._root)
def Button(self, root: _FakeWidget, command: Any | None = None, **_kwargs: Any) -> _FakeButton:
return _FakeButton(root._root, command=command)
def test_show_completion_plays_sound_even_without_overlay_thread(monkeypatch: Any) -> None:
manager = DesktopOverlayManager()
calls: list[str] = []
monkeypatch.setattr(desktop_overlay_module.os, "name", "nt", raising=False)
monkeypatch.setattr(manager, "_play_completion_sound", lambda: calls.append("sound"))
monkeypatch.setattr(manager, "_ensure_thread", lambda: False)
manager.show_completion(
job_id="job-123",
objective="Write a report",
return_message="Finished",
steps=5,
elapsed_seconds=12.4,
)
assert calls == ["sound"]
def test_completion_overlay_auto_dismisses(monkeypatch: Any) -> None:
root = _FakeTk()
fake_tk = _FakeTkModule(root)
monkeypatch.setitem(__import__("sys").modules, "tkinter", fake_tk)
manager = DesktopOverlayManager(auto_dismiss_seconds=0.01)
manager._queue.put(
CompletionOverlayPayload(
job_id="job-123",
objective="Write a report",
return_message="Finished",
steps=5,
elapsed_seconds=12.4,
)
)
manager._ui_main()
assert any(delay == 10 for delay in root.scheduled_delays)
assert root.cards[0]._exists is False
def test_play_completion_sound_uses_winsound_message_beep(monkeypatch: Any) -> None:
calls: list[int] = []
fake_winsound = types.SimpleNamespace(MB_ICONASTERISK=64, MessageBeep=lambda value: calls.append(value))
monkeypatch.setattr(desktop_overlay_module.os, "name", "nt", raising=False)
monkeypatch.setattr(desktop_overlay_module, "winsound", fake_winsound)
DesktopOverlayManager()._play_completion_sound()
assert calls == [64]
tests/test_server_api.py
+15 -90
View File
@@ -7,26 +7,12 @@ from fastapi.testclient import TestClient
import src.server as server_module
from src.config import AppConfig
from src.storage import _objective_category
_TERMINAL_STATUSES = {"completed", "failed", "cancelled"}
def _objective_category(objective: str) -> str:
text = objective.lower()
if any(keyword in text for keyword in ("browser", "website", "amazon", "google", "login", "shopping", "checkout", "orders")):
return "Browser / web"
if any(keyword in text for keyword in ("file", "folder", "directory", "terminal", "shell", "command", "cli", "script", "git", "repo", "install", "pip", "npm")):
return "Files / terminal"
if any(keyword in text for keyword in ("write", "summary", "document", "docs", "report", "email", "message", "readme", "markdown")):
return "Writing / docs"
if any(keyword in text for keyword in ("data", "analysis", "csv", "spreadsheet", "sheet", "table", "chart", "dashboard", "metric", "sql")):
return "Data / analysis"
if any(keyword in text for keyword in ("code", "bug", "fix", "test", "debug", "api", "backend", "frontend", "database", "deploy", "docker", "service", "build")):
return "Development / ops"
return "Other"
class FakeJobManager:
def __init__(self, *, config: AppConfig, db: Any, broadcast: Any = None) -> None:
self.config = config
@@ -94,8 +80,6 @@ class FakeJobManager:
"started_at": created_at,
"ended_at": None,
"steps": 1,
"result": "Running",
"response": {"return": "Running", "data": None},
"return": "Running",
"data": None,
"usage": {
@@ -188,7 +172,6 @@ class FakeJobManager:
def analytics(self) -> dict[str, Any]:
by_category: dict[str, dict[str, Any]] = {}
by_day: dict[str, dict[str, Any]] = {}
def bucket(target: dict[str, dict[str, Any]], key: str) -> dict[str, Any]:
return target.setdefault(
@@ -207,65 +190,28 @@ class FakeJobManager:
},
)
total_jobs = 0
finished_jobs = 0
completed_jobs = 0
failed_jobs = 0
cancelled_jobs = 0
steps_sum = 0
steps_count = 0
cost_sum = 0.0
cost_count = 0
for job in self._jobs.values():
total_jobs += 1
status = str(job.get("status") or "")
finished = status in _TERMINAL_STATUSES
category = _objective_category(str(job.get("objective") or ""))
day = str(job.get("created_at") or "")[:10] or "unknown"
category_bucket = bucket(by_category, category)
day_bucket = bucket(by_day, day)
for item in (category_bucket, day_bucket):
item["total_jobs"] += 1
category_bucket = bucket(by_category, _objective_category(str(job.get("objective") or "")))
category_bucket["total_jobs"] += 1
if not finished:
continue
finished_jobs += 1
category_bucket["finished_jobs"] += 1
if status == "completed":
completed_jobs += 1
category_bucket["completed_jobs"] += 1
elif status == "failed":
failed_jobs += 1
category_bucket["failed_jobs"] += 1
elif status == "cancelled":
cancelled_jobs += 1
category_bucket["cancelled_jobs"] += 1
steps_raw = job.get("steps")
if steps_raw is not None:
steps = int(steps_raw)
steps_sum += steps
steps_count += 1
for item in (category_bucket, day_bucket):
item["steps_sum"] += steps
item["steps_count"] += 1
category_bucket["steps_sum"] += int(steps_raw)
category_bucket["steps_count"] += 1
estimated_cost_raw = (job.get("usage") or {}).get("estimated_cost_usd")
if estimated_cost_raw is not None:
estimated_cost = float(estimated_cost_raw)
cost_sum += estimated_cost
cost_count += 1
for item in (category_bucket, day_bucket):
item["cost_sum"] += estimated_cost
item["cost_count"] += 1
for item in (category_bucket, day_bucket):
item["finished_jobs"] += 1
if status == "completed":
item["completed_jobs"] += 1
elif status == "failed":
item["failed_jobs"] += 1
elif status == "cancelled":
item["cancelled_jobs"] += 1
category_bucket["cost_sum"] += float(estimated_cost_raw)
category_bucket["cost_count"] += 1
def finalize(item: dict[str, Any]) -> dict[str, Any]:
finished = item["finished_jobs"]
@@ -281,18 +227,7 @@ class FakeJobManager:
"avg_cost_usd": round(item["cost_sum"] / item["cost_count"], 6) if item["cost_count"] else None,
}
return {
"total_jobs": total_jobs,
"finished_jobs": finished_jobs,
"completed_jobs": completed_jobs,
"failed_jobs": failed_jobs,
"cancelled_jobs": cancelled_jobs,
"success_rate": round((completed_jobs / finished_jobs) * 100, 2) if finished_jobs else 0.0,
"avg_steps": round(steps_sum / steps_count, 2) if steps_count else None,
"avg_cost_usd": round(cost_sum / cost_count, 6) if cost_count else None,
"by_category": sorted((finalize(item) for item in by_category.values()), key=lambda item: (-item["success_rate"], item["label"])),
"timeline": sorted((finalize(item) for item in by_day.values()), key=lambda item: item["label"]),
}
return {"by_category": sorted((finalize(item) for item in by_category.values()), key=lambda item: item["label"])}
def _build_app(tmp_path: Path, monkeypatch: Any, disable_ui: bool = False):
@@ -352,8 +287,8 @@ def test_create_job_returns_only_job_id_and_defaults_model(tmp_path: Path, monke
status_res = client.get(f"/api/jobs/{job_id}/status", headers=headers)
assert status_res.status_code == 200
assert status_res.json()["job_id"] == job_id
assert status_res.json()["response"]["return"] == "Running"
assert "data" in status_res.json()["response"]
assert status_res.json()["return"] == "Running"
assert status_res.json()["data"] is None
def test_create_job_rejects_invalid_disabled_tool_names(tmp_path: Path, monkeypatch: Any) -> None:
@@ -459,7 +394,7 @@ def test_replay_endpoint_skips_visual_paths_outside_artifacts(tmp_path: Path, mo
assert payload["total_frames"] == 1
def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypatch: Any) -> None:
def test_analytics_endpoint_groups_by_category(tmp_path: Path, monkeypatch: Any) -> None:
app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
manager = app.state.manager
client = TestClient(app)
@@ -495,14 +430,6 @@ def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypa
assert analytics.status_code == 200
payload = analytics.json()
assert payload["total_jobs"] == 3
assert payload["finished_jobs"] == 3
assert payload["completed_jobs"] == 2
assert payload["failed_jobs"] == 1
assert payload["success_rate"] == 66.67
assert payload["avg_steps"] == 6.67
assert payload["avg_cost_usd"] == 0.136667
browser = next(row for row in payload["by_category"] if row["label"] == "Browser / web")
terminal = next(row for row in payload["by_category"] if row["label"] == "Files / terminal")
assert browser["finished_jobs"] == 2
@@ -510,8 +437,6 @@ def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypa
assert browser["avg_steps"] == 5.0
assert terminal["success_rate"] == 100.0
assert [row["label"] for row in payload["timeline"]] == ["2026-05-27", "2026-05-28"]
def test_ui_toggle(tmp_path: Path, monkeypatch: Any) -> None:
app_enabled, _ = _build_app(tmp_path / "enabled", monkeypatch, disable_ui=False)
tests/test_storage.py
+32 -14
View File
@@ -37,8 +37,8 @@ def test_history_db_job_and_events_roundtrip(tmp_path: Path) -> None:
assert job["status"] == "completed"
assert job["model"] == "gpt-5.4-mini"
assert job["disabled_tools"] == ["click"]
assert job["response"]["return"] == "Done"
assert job["response"]["data"]["files"] == ["a.txt", "b.txt"]
assert job["return"] == "Done"
assert job["data"]["files"] == ["a.txt", "b.txt"]
assert job["usage"]["estimated_cost_usd"] == 0.1234
events = db.get_job_events(job_id, limit=10)
@@ -70,11 +70,11 @@ def test_storage_response_fallback_uses_result_when_json_missing(tmp_path: Path)
db.update_job(job_id, status="completed", result="Legacy result string")
job = db.get_job(job_id)
assert job is not None
assert job["response"]["return"] == "Legacy result string"
assert job["response"]["data"] is None
assert job["return"] == "Legacy result string"
assert job["data"] is None
def test_history_db_analytics_groups_by_category_and_day(tmp_path: Path) -> None:
def test_history_db_analytics_groups_by_category(tmp_path: Path) -> None:
db = HistoryDB(tmp_path / "screenjob_test_analytics.db")
db.create_job(
@@ -108,14 +108,6 @@ def test_history_db_analytics_groups_by_category_and_day(tmp_path: Path) -> None
db.update_job("job_terminal_ok", status="completed", steps=10, estimated_cost_usd=0.05)
analytics = db.analytics()
assert analytics["total_jobs"] == 3
assert analytics["finished_jobs"] == 3
assert analytics["completed_jobs"] == 2
assert analytics["failed_jobs"] == 1
assert analytics["success_rate"] == 66.67
assert analytics["avg_steps"] == 6.67
assert analytics["avg_cost_usd"] == 0.136667
browser = next(row for row in analytics["by_category"] if row["label"] == "Browser / web")
terminal = next(row for row in analytics["by_category"] if row["label"] == "Files / terminal")
assert browser["finished_jobs"] == 2
@@ -123,4 +115,30 @@ def test_history_db_analytics_groups_by_category_and_day(tmp_path: Path) -> None
assert browser["avg_steps"] == 5.0
assert terminal["success_rate"] == 100.0
assert [row["label"] for row in analytics["timeline"]] == ["2026-05-27", "2026-05-28"]
def test_prune_older_than_removes_old_terminal_jobs(tmp_path: Path) -> None:
db = HistoryDB(tmp_path / "screenjob_prune.db")
db.create_job(
job_id="job_old",
objective="Old",
model="gpt-5.4-mini",
created_at="2000-01-01T00:00:00+00:00",
safety_override=False,
disabled_tools=[],
)
db.update_job("job_old", status="completed")
db.add_event(job_id="job_old", ts="2000-01-01T00:00:01+00:00", step=1, event_type="done", payload={})
db.create_job(
job_id="job_running",
objective="Running",
model="gpt-5.4-mini",
created_at="2000-01-01T00:00:00+00:00",
safety_override=False,
disabled_tools=[],
)
db.update_job("job_running", status="running")
assert db.prune_older_than(7) == 1
assert db.get_job("job_old") is None
assert db.get_job("job_running") is not None
tests/test_task_manager.py
+48 -37
View File
@@ -4,6 +4,8 @@ import threading
from pathlib import Path
from typing import Any
import pytest
import src.task_manager as task_manager_module
from src.config import AppConfig
from src.models import AgentResult, RunArtifacts, UsageSummary
@@ -11,15 +13,7 @@ from src.storage import HistoryDB
from src.task_manager import JobManager
class _OverlayRecorder:
def __init__(self) -> None:
self.calls: list[dict[str, Any]] = []
def show_completion(self, **kwargs: Any) -> None:
self.calls.append(kwargs)
def _build_manager(tmp_path: Path, overlay_manager: _OverlayRecorder) -> tuple[JobManager, HistoryDB, AppConfig]:
def _build_manager(tmp_path: Path) -> tuple[JobManager, HistoryDB, AppConfig]:
config = AppConfig(
openai_api_key="test-key",
screenjob_token="test-token",
@@ -32,7 +26,7 @@ def _build_manager(tmp_path: Path, overlay_manager: _OverlayRecorder) -> tuple[J
db_path=tmp_path / "screenjob.db",
)
db = HistoryDB(config.db_path)
manager = JobManager(config=config, db=db, overlay_manager=overlay_manager)
manager = JobManager(config=config, db=db)
return manager, db, config
@@ -59,10 +53,9 @@ def _create_job(db: HistoryDB, job_id: str, objective: str) -> None:
)
def test_completed_job_triggers_desktop_overlay(tmp_path: Path, monkeypatch) -> None:
overlay = _OverlayRecorder()
manager, db, _config = _build_manager(tmp_path, overlay)
job_id = "job_overlay_complete"
def test_completed_job_updates_status(tmp_path: Path, monkeypatch) -> None:
manager, db, _config = _build_manager(tmp_path)
job_id = "job_complete"
objective = "Save todo-demo.txt in Documents"
_create_job(db, job_id, objective)
@@ -101,23 +94,16 @@ def test_completed_job_triggers_desktop_overlay(tmp_path: Path, monkeypatch) ->
cancel_event=threading.Event(),
)
assert overlay.calls == [
{
"job_id": job_id,
"objective": objective,
"return_message": "Saved todo-demo.txt",
"steps": 11,
"elapsed_seconds": 12.599999999999994,
}
]
assert db.get_job(job_id)["status"] == "completed"
job = db.get_job(job_id)
assert job is not None
assert job["status"] == "completed"
assert job["return"] == "Saved todo-demo.txt"
def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monkeypatch) -> None:
overlay = _OverlayRecorder()
manager, db, _config = _build_manager(tmp_path, overlay)
def test_non_completed_jobs_are_recorded(tmp_path: Path, monkeypatch) -> None:
manager, db, _config = _build_manager(tmp_path)
failed_job_id = "job_overlay_failed"
failed_job_id = "job_failed"
_create_job(db, failed_job_id, "Fail intentionally")
failed_result = AgentResult(
completed=False,
@@ -131,7 +117,6 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke
error="Failure",
)
monkeypatch.setattr(task_manager_module, "run_job", lambda **_kwargs: (failed_result, _artifacts(tmp_path)))
manager._execute_job(
job_id=failed_job_id,
objective="Fail intentionally",
@@ -155,7 +140,7 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke
cancel_event=threading.Event(),
)
cancelled_job_id = "job_overlay_cancelled"
cancelled_job_id = "job_cancelled"
_create_job(db, cancelled_job_id, "Cancel intentionally")
cancelled_result = AgentResult(
completed=False,
@@ -170,7 +155,6 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke
cancelled=True,
)
monkeypatch.setattr(task_manager_module, "run_job", lambda **_kwargs: (cancelled_result, _artifacts(tmp_path)))
manager._execute_job(
job_id=cancelled_job_id,
objective="Cancel intentionally",
@@ -194,13 +178,13 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke
cancel_event=threading.Event(),
)
assert overlay.calls == []
assert db.get_job(failed_job_id)["status"] == "failed"
assert db.get_job(cancelled_job_id)["status"] == "cancelled"
def test_rejected_job_does_not_trigger_desktop_overlay(tmp_path: Path, monkeypatch) -> None:
overlay = _OverlayRecorder()
manager, db, _config = _build_manager(tmp_path, overlay)
job_id = "job_overlay_rejected"
def test_rejected_job_is_recorded(tmp_path: Path, monkeypatch) -> None:
manager, db, _config = _build_manager(tmp_path)
job_id = "job_rejected"
_create_job(db, job_id, "Do something unsafe")
monkeypatch.setattr(task_manager_module, "create_openai_client", lambda *_args, **_kwargs: object())
@@ -233,6 +217,33 @@ def test_rejected_job_does_not_trigger_desktop_overlay(tmp_path: Path, monkeypat
cancel_event=threading.Event(),
)
assert overlay.calls == []
events = db.get_job_events(job_id)
assert events[-1]["event_type"] == "job_rejected"
def test_submit_job_rejects_when_another_run_is_active(tmp_path: Path) -> None:
manager, _db, _config = _build_manager(tmp_path)
ready = threading.Event()
release = threading.Event()
def _hold() -> None:
ready.set()
release.wait()
active_thread = threading.Thread(target=_hold)
active_thread.start()
ready.wait(1)
manager._running["job_active"] = task_manager_module._RunningJob(
thread=active_thread,
cancel_event=threading.Event(),
started_at="2026-05-30T12:00:00+00:00",
objective="Active",
model="gpt-5.4-mini",
)
try:
with pytest.raises(ValueError, match="already active"):
manager.submit_job(objective="Second run")
finally:
release.set()
active_thread.join(timeout=1)