feat: implement replay functionality with UI controls and backend support

tests/test_server_api.py

@@ -33,6 +33,10 @@ class FakeJobManager:
         self._counter += 1
         job_id = f"job_fake_{self._counter:03d}"
         selected_model = (model or self.config.default_model).strip()
+        artifacts_dir = (self.config.runs_dir / f"run_{job_id}").resolve()
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+        screenshot_path = artifacts_dir / "screen_step_001.png"
+        screenshot_path.write_bytes(b"not-a-real-png")
         self.last_submit_payload = {
             "objective": objective,
             "model": selected_model,
@@ -61,7 +65,7 @@ class FakeJobManager:
                 "total_tokens": 14,
                 "estimated_cost_usd": 0.0001,
             },
-            "artifacts_dir": str(self.config.runs_dir.resolve()),
+            "artifacts_dir": str(artifacts_dir),
         }
         self._events[job_id] = [
             {
@@ -70,7 +74,47 @@ class FakeJobManager:
                 "ts": "2026-05-27T00:00:00Z",
                 "step": 1,
                 "event_type": "tool_called",
-                "payload": {"tool": "execute_command"},
+                "payload": {"tool": "click", "args": {"coordinate": {"x": 320, "y": 180}}},
+            },
+            {
+                "id": 2,
+                "job_id": job_id,
+                "ts": "2026-05-27T00:00:01Z",
+                "step": 1,
+                "event_type": "tool_result",
+                "payload": {"tool": "click", "result": {"ok": True, "clicked": {"x": 322, "y": 182}}},
+            },
+            {
+                "id": 3,
+                "job_id": job_id,
+                "ts": "2026-05-27T00:00:02Z",
+                "step": 1,
+                "event_type": "tool_called",
+                "payload": {"tool": "type", "args": {"text": "hello world"}},
+            },
+            {
+                "id": 4,
+                "job_id": job_id,
+                "ts": "2026-05-27T00:00:03Z",
+                "step": 1,
+                "event_type": "tool_result",
+                "payload": {"tool": "type", "result": {"ok": True, "typed_length": 11}},
+            },
+            {
+                "id": 5,
+                "job_id": job_id,
+                "ts": "2026-05-27T00:00:04Z",
+                "step": 1,
+                "event_type": "visual_update",
+                "payload": {
+                    "kind": "see_screen",
+                    "image_meta": {
+                        "path": str(screenshot_path),
+                        "width": 1920,
+                        "height": 1080,
+                        "grid": True,
+                    },
+                },
             }
         ]
         return job_id
@@ -174,6 +218,58 @@ def test_cancel_endpoint_and_events(tmp_path: Path, monkeypatch: Any) -> None:
     assert status_after["data"] is None
 
 
+def test_replay_endpoint_builds_frames_and_overlays(tmp_path: Path, monkeypatch: Any) -> None:
+    app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
+    client = TestClient(app)
+    headers = {"Authorization": "Bearer test_token"}
+    create = client.post("/api/jobs", headers=headers, json={"job": "Replay test"})
+    job_id = create.json()["job_id"]
+
+    replay = client.get(f"/api/jobs/{job_id}/replay?limit=200", headers=headers)
+    assert replay.status_code == 200
+    payload = replay.json()
+    assert payload["job_id"] == job_id
+    assert payload["total_frames"] == 1
+    frame = payload["frames"][0]
+    assert frame["kind"] == "see_screen"
+    assert frame["is_fullscreen"] is True
+    labels = [item.get("label", "") for item in frame["overlays"]]
+    assert any("click" in text.lower() for text in labels)
+    assert any("typed" in text.lower() for text in labels)
+
+
+def test_replay_endpoint_skips_visual_paths_outside_artifacts(tmp_path: Path, monkeypatch: Any) -> None:
+    app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
+    manager = app.state.manager
+    client = TestClient(app)
+    headers = {"Authorization": "Bearer test_token"}
+    create = client.post("/api/jobs", headers=headers, json={"job": "Replay path check"})
+    job_id = create.json()["job_id"]
+    manager._events[job_id].append(
+        {
+            "id": 999,
+            "job_id": job_id,
+            "ts": "2026-05-27T00:01:00Z",
+            "step": 2,
+            "event_type": "visual_update",
+            "payload": {
+                "kind": "see_screen",
+                "image_meta": {
+                    "path": str((tmp_path / "outside.png").resolve()),
+                    "width": 100,
+                    "height": 100,
+                    "grid": True,
+                },
+            },
+        }
+    )
+
+    replay = client.get(f"/api/jobs/{job_id}/replay?limit=500", headers=headers)
+    assert replay.status_code == 200
+    payload = replay.json()
+    assert payload["total_frames"] == 1
+
+
 def test_ui_toggle(tmp_path: Path, monkeypatch: Any) -> None:
     app_enabled, _ = _build_app(tmp_path / "enabled", monkeypatch, disable_ui=False)
     client_enabled = TestClient(app_enabled)