space/screenjob
1
0
Fork 0
Code Issues Pull Requests Actions Releases Activity

feat. Revision of Agent Lopp & major clean-up
CI / test (push) Successful in 8s
Details

Browse Source
This commit is contained in:
Space
2026-06-04 18:00:26 +02:00
parent 1f606475c5
commit 75a2600e4d
28 changed files with 753 additions and 1941 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_agent_tools.py
+93 -85
View File
@@ -343,20 +343,20 @@ def test_context_compaction_trigger_and_payload(tmp_path: Path, monkeypatch) ->
agent.step = 4
agent.last_context_compact_step = 0
agent.options.screen_context_decay_steps = 4
agent.recent_tool_summaries = ["step=1 tool=see_screen status=ok"]
agent.recent_tool_summaries = ["step=1 tool=enhance status=ok"]
agent.last_screen_data_url = "data:image/png;base64,abc"
agent.last_screen_meta = {"width": 1280, "height": 720, "path": "C:/tmp/frame.png"}
assert agent._should_compact_context() is True
visual_message = agent._build_visual_message("Current screen", "data:image/png;base64,abc", agent.last_screen_meta)
agent._register_visual_context_message(visual_message, agent.last_screen_meta, tool_name="see_screen")
agent._register_visual_context_message(visual_message, agent.last_screen_meta, tool_name="enhance")
compacted = agent._build_compacted_pending_input("decay")
assert len(compacted) == 2
assert "Context compaction activated due to stale context decay." in compacted[0]["content"][0]["text"]
assert "Open settings app" in compacted[0]["content"][0]["text"]
assert "Treat prior reasoning as stale" in compacted[0]["content"][0]["text"]
assert "Retained visual observations:" in compacted[0]["content"][0]["text"]
assert "do not call see_screen again only because compaction happened" in compacted[0]["content"][0]["text"]
assert "do not ask for another visual just because compaction happened" in compacted[0]["content"][0]["text"]
assert "observe -> decide -> act -> verify" in compacted[0]["content"][0]["text"]
@@ -365,7 +365,7 @@ def test_context_compaction_drops_function_call_outputs_from_rebased_input(tmp_p
agent.objective = "Open settings app"
visual_meta = {"path": "C:/tmp/frame.png"}
visual_message = agent._build_visual_message("Current screen", "data:image/png;base64,abc", visual_meta)
agent._register_visual_context_message(visual_message, visual_meta, tool_name="see_screen")
agent._register_visual_context_message(visual_message, visual_meta, tool_name="enhance")
compacted = agent._build_compacted_pending_input(
"decay",
@@ -390,17 +390,36 @@ def test_visual_context_budget_keeps_only_latest_three_images(tmp_path: Path, mo
"2026-05-30T10:00:01+00:00",
"2026-05-30T10:00:04+00:00",
"2026-05-30T10:00:02+00:00",
"2026-05-30T10:00:05+00:00",
]
for idx, captured_at in enumerate(captured_times):
meta = {"path": f"C:/tmp/frame_{idx}.png", "captured_at": captured_at}
message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta)
agent._register_visual_context_message(message, meta, tool_name="see_screen")
agent._register_visual_context_message(message, meta, tool_name="enhance")
assert agent.visual_context_overflow_pending is True
assert [entry["meta"]["path"] for entry in agent.visual_context_messages] == [
"C:/tmp/frame_3.png",
"C:/tmp/frame_0.png",
"C:/tmp/frame_2.png",
"C:/tmp/frame_4.png",
]
def test_visual_context_budget_does_not_overflow_on_small_headroom(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.options.max_visual_context_images = 3
for idx in range(4):
meta = {"path": f"C:/tmp/frame_{idx}.png"}
message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta)
agent._register_visual_context_message(message, meta, tool_name="enhance")
assert agent.visual_context_overflow_pending is False
assert [entry["meta"]["path"] for entry in agent.visual_context_messages] == [
"C:/tmp/frame_0.png",
"C:/tmp/frame_1.png",
"C:/tmp/frame_2.png",
"C:/tmp/frame_3.png",
]
@@ -419,7 +438,7 @@ def test_compacted_input_uses_latest_visuals_by_capture_time(tmp_path: Path, mon
):
meta = {"path": f"C:/tmp/frame_{idx}.png", "captured_at": captured_at}
message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta)
agent._register_visual_context_message(message, meta, tool_name="see_screen")
agent._register_visual_context_message(message, meta, tool_name="enhance")
compacted = agent._build_compacted_pending_input("visual_budget")
visual_messages = [
@@ -460,33 +479,47 @@ def test_context_compaction_event_includes_visual_budget_reason_and_paths(tmp_pa
assert payload["visual_context_paths"] == ["C:/tmp/1.png", "C:/tmp/2.png", "C:/tmp/3.png"]
def test_context_compaction_log_uses_readable_reason(tmp_path: Path, monkeypatch, caplog) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.step = 7
agent.visual_context_messages = [
{"message": {"role": "user", "content": []}, "meta": {"path": "C:/tmp/1.png"}},
]
with caplog.at_level(logging.INFO, logger=agent.logger.name):
agent._emit_context_compacted("visual_budget")
assert "Context compacted at step 7 (reason=visual budget overflow, retained_visuals=1)" in caplog.text
assert "visual_budget" not in caplog.text
def test_observation_loop_blocks_repeated_broad_reobservation(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.step_history = [
{
"step": 21,
"tool_names": ["get_active_window", "see_screen"],
"tool_names": ["get_active_window", "enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
},
{
"step": 22,
"tool_names": ["get_active_window", "see_screen"],
"tool_names": ["get_active_window", "enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
},
{
"step": 23,
"tool_names": ["get_active_window", "see_screen"],
"tool_names": ["get_active_window", "enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
},
]
blocked = agent._dispatch_tool("see_screen", {})
blocked = agent._dispatch_tool("enhance", {})
assert blocked["ok"] is False
assert blocked["blocked"] is True
@@ -508,7 +541,7 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat
},
{
"step": 41,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
@@ -540,7 +573,7 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat
},
{
"step": 45,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "123|#32770|Save as",
"window_summary": "Save as [#32770]",
"had_visual": True,
@@ -548,12 +581,9 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat
},
]
blocked = agent._dispatch_tool("see_screen", {})
stable = agent._stable_observation_loop()
assert blocked["ok"] is False
assert blocked["blocked"] is True
assert blocked["blocked_reason"] == "observation_loop"
assert blocked["repeated_steps"] == 3
assert stable is None or stable["window_summary"] == "Save as [#32770]"
def test_observation_loop_treats_focus_window_as_action_progress(tmp_path: Path, monkeypatch) -> None:
@@ -570,7 +600,7 @@ def test_observation_loop_requires_non_empty_window_signature(tmp_path: Path, mo
agent.step_history = [
{
"step": 50,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "",
"window_summary": "",
"had_visual": True,
@@ -586,7 +616,7 @@ def test_observation_loop_requires_non_empty_window_signature(tmp_path: Path, mo
},
{
"step": 52,
"tool_names": ["see_screen"],
"tool_names": ["enhance"],
"window_signature": "",
"window_summary": "",
"had_visual": True,
@@ -606,15 +636,11 @@ def test_record_step_history_reuses_last_observed_window_for_visual_only_steps(t
"title": "Settings",
}
agent._record_step_history(["see_screen"], None, "sig-a")
agent._record_step_history(["get_active_window"], None)
agent._record_step_history(["detect_dialog"], None)
agent._record_step_history(["enhance"], None, "sig-a")
stable = agent._stable_observation_loop()
assert stable is not None
assert stable["window_summary"] == "Settings [ApplicationFrameWindow]"
assert stable["repeated_steps"] == 3
entry = agent.step_history[-1]
assert entry["window_summary"] == "Settings [ApplicationFrameWindow]"
assert entry["window_signature"]
def test_repeated_ambiguous_action_requires_verification_and_then_blocks(tmp_path: Path, monkeypatch) -> None:
@@ -622,27 +648,12 @@ def test_repeated_ambiguous_action_requires_verification_and_then_blocks(tmp_pat
type_args = {"text": "repeat me"}
first = agent._dispatch_tool("type", type_args)
second = agent._dispatch_tool("type", type_args)
third = agent._dispatch_tool("type", type_args)
assert first["ok"] is True
assert first["verification_required"] is True
assert first["verification_channels"] == ["enhance", "get_active_window", "see_screen"]
blocked_without_verification = agent._dispatch_tool("type", type_args)
assert blocked_without_verification["blocked"] is True
assert "see_screen" in blocked_without_verification["error"]
assert agent._dispatch_tool("see_screen", {})["ok"] is True
assert agent._dispatch_tool("type", type_args)["ok"] is True
assert agent._dispatch_tool("see_screen", {})["ok"] is True
assert agent._dispatch_tool("type", type_args)["ok"] is True
assert agent._dispatch_tool("see_screen", {})["ok"] is True
blocked_after_retry_budget = agent._dispatch_tool("type", type_args)
assert blocked_after_retry_budget["blocked"] is True
assert "3 time(s) on the same surface" in blocked_after_retry_budget["error"]
assert agent._dispatch_tool("see_screen", {})["ok"] is True
reset_attempt = agent._dispatch_tool("type", type_args)
assert reset_attempt["ok"] is True
assert second["ok"] is True
assert third["ok"] is True
def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatch) -> None:
@@ -656,7 +667,7 @@ def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatc
first = agent._dispatch_tool("press_key", {"key": "ctrl+c"})
assert first["ok"] is True
assert first["verification_channels"] == ["clipboard_get"]
assert "verification_channels" not in first
blocked = agent._dispatch_tool("press_key", {"key": "ctrl+c"})
assert blocked["blocked"] is True
@@ -670,6 +681,22 @@ def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatc
assert second["ok"] is True
def test_sleep_requires_a_previous_tool_call(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
blocked = agent._dispatch_tool("sleep", {"seconds": 0.1})
assert blocked["ok"] is False
assert blocked["blocked"] is True
assert blocked["blocked_reason"] == "sleep_requires_previous_tool_call"
observed = agent._dispatch_tool("get_cursor_position", {})
assert observed["ok"] is True
allowed = agent._dispatch_tool("sleep", {"seconds": 0.1})
assert allowed["ok"] is True
assert allowed["slept_seconds"] == 0.1
def test_execute_command_blocks_unrequested_recursive_file_search(tmp_path: Path, monkeypatch) -> None:
agent = _build_agent(tmp_path, monkeypatch)
agent.objective = "Save the current note in Notepad"
@@ -732,15 +759,15 @@ def test_execute_command_launch_requires_focus_verification(tmp_path: Path, monk
assert first["ok"] is True
assert first["background_launch_assumed"] is True
assert first["focus_change_assumed"] is False
assert first["verification_required"] is True
assert first["verification_channels"] == ["get_active_window", "see_screen"]
assert "verification_required" not in first
assert "verification_channels" not in first
assert called["command"] == "start notepad"
blocked = agent._dispatch_tool("execute_command", {"command": "start notepad"})
assert blocked["blocked"] is True
assert "get_active_window" in blocked["error"]
assert "enhance" in blocked["error"]
observed = agent._dispatch_tool("get_active_window", {})
observed = agent._dispatch_tool("enhance", {})
assert observed["ok"] is True
second = agent._dispatch_tool("execute_command", {"command": "start notepad"})
@@ -750,21 +777,11 @@ def test_execute_command_launch_requires_focus_verification(tmp_path: Path, monk
def test_system_prompt_emphasizes_situational_awareness() -> None:
prompt = agent_module.SYSTEM_PROMPT
assert "Maintain a live mental model" in prompt
assert "classify -> choose control channel -> execute one meaningful transition -> verify" in prompt
assert "First classify, then act." in prompt
assert "Use see_screen at a balanced cadence" in prompt
assert "get_active_window" in prompt
assert "detect_dialog" in prompt
assert "dialog_set_filename" in prompt
assert "list_ui_elements" in prompt
assert "clipboard_get" in prompt
assert "Do not invent new subgoals" in prompt
assert "verify-and-finish" in prompt
assert "Use tools to act" in prompt
assert "observe -> choose the best tool -> make one meaningful move -> verify" in prompt
assert "Do not assume command-launched apps or URLs became foreground" in prompt
assert "Resolve unexpected modals before resuming the old plan" in prompt
assert "data.observed_result" in prompt
assert "Treat command-launched apps or URLs as background" in prompt
assert "#32770" in prompt
assert "secure desktop" in prompt.lower()
def test_observation_loop_prompt_pushes_action_or_finish() -> None:
@@ -786,7 +803,7 @@ def test_finish_likely_prompt_pushes_verification_then_completion() -> None:
assert "objective is likely already satisfied" in prompt
assert "todo-demo.txt - Notepad" in prompt
assert "call see_screen" in prompt
assert "add enhance only if the proof is small or text-heavy" in prompt
assert "then call task_complete" in prompt
assert "Do not reopen menus" in prompt
assert "Prohibited key combos for this run: ctrl+shift+s." in prompt
@@ -800,12 +817,11 @@ def test_initial_action_prompt_reinforces_observation_and_verification() -> None
assert "Identify what changed since the last action or screen capture." in prompt
assert "classify -> choose control channel -> execute one meaningful transition -> verify" in prompt
assert "Prefer native window/dialog/element tools" in prompt
assert "get_active_window plus detect_dialog" in prompt
assert "click then see_screen" in prompt
assert "Do not invent new subgoals" in prompt
assert "Prefer non-visual verification when available" in prompt
assert "wait_for_focus_change" in prompt
assert "#32770 dialogs" in prompt
assert "verify the expected UI or focus change before repeating the same action or chaining another risky action" in prompt
assert "Prohibited key combos for this run: ctrl+shift+s." in prompt
assert "do not re-capture the screen just to reconfirm an obvious large input area" in prompt
assert 'task_complete(return=..., data={"observed_result": ...})' in prompt
@@ -816,7 +832,6 @@ def test_no_tool_prompt_recovers_by_reobserving() -> None:
assert "Recover by re-observing the current desktop state instead of guessing." in prompt
assert "Start by classifying the surface." in prompt
assert "get_active_window" in prompt
assert "detect_dialog" in prompt
assert "clipboard_get" in prompt
assert "native window/dialog/element tools" in prompt
@@ -833,9 +848,7 @@ def test_blocked_action_prompt_reanchors_on_screen_state() -> None:
assert "classify the current surface" in prompt
assert "detect_dialog" in prompt
assert "dialog_set_filename" in prompt
assert "get_active_window" in prompt
assert "get_cursor_position before move_mouse or drag" in prompt
assert "wait_for_focus_change" in prompt
assert "secure desktop or UAC" in prompt
assert "Switch strategy after the fresh classification" in prompt
assert "Prohibited key combos for this run: ctrl+shift+s." in prompt
@@ -848,16 +861,13 @@ def test_tool_schemas_include_completion_and_desktop_awareness_guidance(tmp_path
schemas = {tool["name"]: tool for tool in agent._tool_schemas()}
assert "data.observed_result" in schemas["task_complete"]["description"]
assert "before task_complete" in schemas["see_screen"]["description"]
assert "text-heavy targets" in schemas["enhance"]["description"]
assert "verify copy or cut results" in schemas["clipboard_get"]["description"]
assert "pointer state matters" in schemas["get_cursor_position"]["description"]
assert "verify focus and active app" in schemas["get_active_window"]["description"]
assert "text-heavy" in schemas["enhance"]["description"]
assert "copy or cut" in schemas["clipboard_get"]["description"]
assert "pointer" in schemas["get_cursor_position"]["description"]
assert "foreground focus" in schemas["execute_command"]["description"]
assert "Prohibited for this run: ctrl+shift+s." in schemas["press_key"]["description"]
assert "dialog classification" in schemas["get_active_window"]["description"]
assert "visible top-level windows" in schemas["list_windows"]["description"]
assert "#32770 or picker surface" in schemas["detect_dialog"]["description"]
assert "#32770" in schemas["detect_dialog"]["description"]
assert "filename or path field" in schemas["dialog_set_filename"]["description"]
assert "native child controls" in schemas["list_ui_elements"]["description"]
@@ -868,7 +878,6 @@ def test_tool_schemas_hide_optional_native_tools_when_mode_off(tmp_path: Path, m
schemas = {tool["name"]: tool for tool in agent._tool_schemas()}
assert "get_active_window" in schemas
assert "list_windows" not in schemas
assert "detect_dialog" not in schemas
assert "list_ui_elements" not in schemas
@@ -880,15 +889,14 @@ def test_tool_schemas_hide_windows_only_tools_on_non_windows_host(tmp_path: Path
schemas = {tool["name"]: tool for tool in agent._tool_schemas()}
assert "get_active_window" not in schemas
assert "list_windows" not in schemas
assert "detect_dialog" not in schemas
assert "list_ui_elements" not in schemas
result = agent._dispatch_tool("get_active_window", {})
result = agent._dispatch_tool("list_windows", {})
assert result["ok"] is False
assert result["error"] == "Tool 'get_active_window' is only available on Windows."
assert result["error"] == "Tool 'list_windows' is only available on Windows."
def test_list_windows_returns_structured_surface_metadata(tmp_path: Path, monkeypatch) -> None:
@@ -1041,7 +1049,7 @@ def test_finish_likely_guard_blocks_reopening_menu_after_fresh_verification(tmp_
)
agent.step = 25
verify_result = agent._dispatch_tool("see_screen", {})
verify_result = agent._dispatch_tool("enhance", {})
assert verify_result["ok"] is True
assert verify_result["finish_likely_verification_done"] is True
assert agent.finish_likely_state["fresh_verification_done"] is True