291 lines
11 KiB
Python
291 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
import src.server as server_module
|
|
from src.config import AppConfig
|
|
|
|
|
|
class FakeJobManager:
|
|
def __init__(self, *, config: AppConfig, db: Any, broadcast: Any = None) -> None:
|
|
self.config = config
|
|
self._jobs: dict[str, dict[str, Any]] = {}
|
|
self._events: dict[str, list[dict[str, Any]]] = {}
|
|
self._counter = 0
|
|
self.last_submit_payload: dict[str, Any] | None = None
|
|
|
|
def submit_job(
|
|
self,
|
|
*,
|
|
objective: str,
|
|
model: str | None = None,
|
|
max_steps: int = 60,
|
|
command_timeout: int = 45,
|
|
type_interval: float = 0.02,
|
|
click_pause: float = 0.10,
|
|
reasoning_effort: str = "medium",
|
|
disabled_tools: list[str] | None = None,
|
|
safety_override: bool = False,
|
|
no_failsafe: bool = False,
|
|
) -> str:
|
|
self._counter += 1
|
|
job_id = f"job_fake_{self._counter:03d}"
|
|
selected_model = (model or self.config.default_model).strip()
|
|
artifacts_dir = (self.config.runs_dir / f"run_{job_id}").resolve()
|
|
artifacts_dir.mkdir(parents=True, exist_ok=True)
|
|
screenshot_path = artifacts_dir / "screen_step_001.png"
|
|
screenshot_path.write_bytes(b"not-a-real-png")
|
|
self.last_submit_payload = {
|
|
"objective": objective,
|
|
"model": selected_model,
|
|
"disabled_tools": disabled_tools or [],
|
|
"safety_override": safety_override,
|
|
"max_steps": max_steps,
|
|
"command_timeout": command_timeout,
|
|
"type_interval": type_interval,
|
|
"click_pause": click_pause,
|
|
"reasoning_effort": reasoning_effort,
|
|
"no_failsafe": no_failsafe,
|
|
}
|
|
self._jobs[job_id] = {
|
|
"job_id": job_id,
|
|
"objective": objective,
|
|
"model": selected_model,
|
|
"status": "running",
|
|
"result": "Running",
|
|
"response": {"return": "Running", "data": None},
|
|
"return": "Running",
|
|
"data": None,
|
|
"usage": {
|
|
"input_tokens": 10,
|
|
"cached_input_tokens": 2,
|
|
"output_tokens": 4,
|
|
"reasoning_tokens": 0,
|
|
"total_tokens": 14,
|
|
"estimated_cost_usd": 0.0001,
|
|
},
|
|
"artifacts_dir": str(artifacts_dir),
|
|
}
|
|
self._events[job_id] = [
|
|
{
|
|
"id": 1,
|
|
"job_id": job_id,
|
|
"ts": "2026-05-27T00:00:00Z",
|
|
"step": 1,
|
|
"event_type": "tool_called",
|
|
"payload": {"tool": "click", "args": {"coordinate": {"x": 320, "y": 180}}},
|
|
},
|
|
{
|
|
"id": 2,
|
|
"job_id": job_id,
|
|
"ts": "2026-05-27T00:00:01Z",
|
|
"step": 1,
|
|
"event_type": "tool_result",
|
|
"payload": {"tool": "click", "result": {"ok": True, "clicked": {"x": 322, "y": 182}}},
|
|
},
|
|
{
|
|
"id": 3,
|
|
"job_id": job_id,
|
|
"ts": "2026-05-27T00:00:02Z",
|
|
"step": 1,
|
|
"event_type": "tool_called",
|
|
"payload": {"tool": "type", "args": {"text": "hello world"}},
|
|
},
|
|
{
|
|
"id": 4,
|
|
"job_id": job_id,
|
|
"ts": "2026-05-27T00:00:03Z",
|
|
"step": 1,
|
|
"event_type": "tool_result",
|
|
"payload": {"tool": "type", "result": {"ok": True, "typed_length": 11}},
|
|
},
|
|
{
|
|
"id": 5,
|
|
"job_id": job_id,
|
|
"ts": "2026-05-27T00:00:04Z",
|
|
"step": 1,
|
|
"event_type": "visual_update",
|
|
"payload": {
|
|
"kind": "see_screen",
|
|
"image_meta": {
|
|
"path": str(screenshot_path),
|
|
"width": 1920,
|
|
"height": 1080,
|
|
"grid": True,
|
|
},
|
|
},
|
|
}
|
|
]
|
|
return job_id
|
|
|
|
def list_jobs(self, limit: int = 100) -> list[dict[str, Any]]:
|
|
return list(self._jobs.values())[:limit]
|
|
|
|
def get_job(self, job_id: str) -> dict[str, Any] | None:
|
|
return self._jobs.get(job_id)
|
|
|
|
def get_events(self, job_id: str, limit: int = 500) -> list[dict[str, Any]]:
|
|
return self._events.get(job_id, [])[:limit]
|
|
|
|
def cancel_job(self, job_id: str) -> bool:
|
|
if job_id not in self._jobs:
|
|
return False
|
|
self._jobs[job_id]["status"] = "cancelling"
|
|
return True
|
|
|
|
def stats(self) -> dict[str, Any]:
|
|
return {
|
|
"total_jobs": len(self._jobs),
|
|
"running_jobs": sum(1 for x in self._jobs.values() if x["status"] == "running"),
|
|
"completed_jobs": 0,
|
|
"failed_jobs": 0,
|
|
"cancelled_jobs": 0,
|
|
"total_estimated_cost": sum(float((x["usage"] or {}).get("estimated_cost_usd") or 0) for x in self._jobs.values()),
|
|
"live_running_threads": 0,
|
|
}
|
|
|
|
|
|
def _build_app(tmp_path: Path, monkeypatch: Any, disable_ui: bool = False):
|
|
monkeypatch.setattr(server_module, "JobManager", FakeJobManager)
|
|
config = AppConfig(
|
|
openai_api_key="test_key",
|
|
screenjob_token="test_token",
|
|
disable_ui=disable_ui,
|
|
default_model="gpt-5.4-mini",
|
|
safety_model="gpt-5.4-mini",
|
|
host="127.0.0.1",
|
|
port=8787,
|
|
runs_dir=tmp_path / "runs",
|
|
db_path=tmp_path / "screenjob_test.db",
|
|
)
|
|
config.runs_dir.mkdir(parents=True, exist_ok=True)
|
|
app = server_module.create_app(config)
|
|
return app, config
|
|
|
|
|
|
def test_api_requires_auth(tmp_path: Path, monkeypatch: Any) -> None:
|
|
app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
|
|
client = TestClient(app)
|
|
assert client.get("/api/jobs").status_code == 401
|
|
assert client.post("/api/jobs", json={"job": "x"}).status_code == 401
|
|
|
|
|
|
def test_create_job_returns_only_job_id_and_defaults_model(tmp_path: Path, monkeypatch: Any) -> None:
|
|
app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
|
|
client = TestClient(app)
|
|
headers = {"Authorization": "Bearer test_token"}
|
|
|
|
response = client.post(
|
|
"/api/jobs",
|
|
headers=headers,
|
|
json={"job": "Open amazon.de", "disabled_tools": ["click"], "safety_override": True},
|
|
)
|
|
assert response.status_code == 200
|
|
payload = response.json()
|
|
assert list(payload.keys()) == ["job_id"]
|
|
job_id = payload["job_id"]
|
|
|
|
manager = app.state.manager
|
|
assert manager.last_submit_payload["model"] == "gpt-5.4-mini"
|
|
assert manager.last_submit_payload["disabled_tools"] == ["click"]
|
|
assert manager.last_submit_payload["reasoning_effort"] == "medium"
|
|
|
|
status_res = client.get(f"/api/jobs/{job_id}/status", headers=headers)
|
|
assert status_res.status_code == 200
|
|
assert status_res.json()["job_id"] == job_id
|
|
assert status_res.json()["response"]["return"] == "Running"
|
|
assert "data" in status_res.json()["response"]
|
|
|
|
|
|
def test_cancel_endpoint_and_events(tmp_path: Path, monkeypatch: Any) -> None:
|
|
app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
|
|
client = TestClient(app)
|
|
headers = {"Authorization": "Bearer test_token"}
|
|
create = client.post("/api/jobs", headers=headers, json={"job": "Test job"})
|
|
job_id = create.json()["job_id"]
|
|
|
|
events = client.get(f"/api/jobs/{job_id}/events?limit=20", headers=headers)
|
|
assert events.status_code == 200
|
|
assert len(events.json()["events"]) >= 1
|
|
|
|
cancel = client.post(f"/api/jobs/{job_id}/cancel", headers=headers)
|
|
assert cancel.status_code == 200
|
|
assert cancel.json()["cancel_requested"] is True
|
|
|
|
status_after = client.get(f"/api/jobs/{job_id}", headers=headers).json()
|
|
assert status_after["status"] == "cancelling"
|
|
assert status_after["return"] == "Running"
|
|
assert status_after["data"] is None
|
|
|
|
|
|
def test_replay_endpoint_builds_frames_and_overlays(tmp_path: Path, monkeypatch: Any) -> None:
|
|
app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
|
|
client = TestClient(app)
|
|
headers = {"Authorization": "Bearer test_token"}
|
|
create = client.post("/api/jobs", headers=headers, json={"job": "Replay test"})
|
|
job_id = create.json()["job_id"]
|
|
|
|
replay = client.get(f"/api/jobs/{job_id}/replay?limit=200", headers=headers)
|
|
assert replay.status_code == 200
|
|
payload = replay.json()
|
|
assert payload["job_id"] == job_id
|
|
assert payload["total_frames"] == 1
|
|
frame = payload["frames"][0]
|
|
assert frame["kind"] == "see_screen"
|
|
assert frame["is_fullscreen"] is True
|
|
labels = [item.get("label", "") for item in frame["overlays"]]
|
|
assert any("click" in text.lower() for text in labels)
|
|
assert any("typed" in text.lower() for text in labels)
|
|
|
|
|
|
def test_replay_endpoint_skips_visual_paths_outside_artifacts(tmp_path: Path, monkeypatch: Any) -> None:
|
|
app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
|
|
manager = app.state.manager
|
|
client = TestClient(app)
|
|
headers = {"Authorization": "Bearer test_token"}
|
|
create = client.post("/api/jobs", headers=headers, json={"job": "Replay path check"})
|
|
job_id = create.json()["job_id"]
|
|
manager._events[job_id].append(
|
|
{
|
|
"id": 999,
|
|
"job_id": job_id,
|
|
"ts": "2026-05-27T00:01:00Z",
|
|
"step": 2,
|
|
"event_type": "visual_update",
|
|
"payload": {
|
|
"kind": "see_screen",
|
|
"image_meta": {
|
|
"path": str((tmp_path / "outside.png").resolve()),
|
|
"width": 100,
|
|
"height": 100,
|
|
"grid": True,
|
|
},
|
|
},
|
|
}
|
|
)
|
|
|
|
replay = client.get(f"/api/jobs/{job_id}/replay?limit=500", headers=headers)
|
|
assert replay.status_code == 200
|
|
payload = replay.json()
|
|
assert payload["total_frames"] == 1
|
|
|
|
|
|
def test_ui_toggle(tmp_path: Path, monkeypatch: Any) -> None:
|
|
app_enabled, _ = _build_app(tmp_path / "enabled", monkeypatch, disable_ui=False)
|
|
client_enabled = TestClient(app_enabled)
|
|
root_enabled = client_enabled.get("/")
|
|
assert root_enabled.status_code == 200
|
|
assert "ScreenJob Monitor" in root_enabled.text
|
|
js_enabled = client_enabled.get("/ui/monitoring.js")
|
|
assert js_enabled.status_code == 200
|
|
assert "const tokenInput" in js_enabled.text
|
|
|
|
app_disabled, _ = _build_app(tmp_path / "disabled", monkeypatch, disable_ui=True)
|
|
client_disabled = TestClient(app_disabled)
|
|
root_disabled = client_disabled.get("/")
|
|
assert root_disabled.status_code == 200
|
|
assert root_disabled.json()["ui_disabled"] is True
|