This commit is contained in:
+15
-90
@@ -7,26 +7,12 @@ from fastapi.testclient import TestClient
|
||||
|
||||
import src.server as server_module
|
||||
from src.config import AppConfig
|
||||
from src.storage import _objective_category
|
||||
|
||||
|
||||
_TERMINAL_STATUSES = {"completed", "failed", "cancelled"}
|
||||
|
||||
|
||||
def _objective_category(objective: str) -> str:
|
||||
text = objective.lower()
|
||||
if any(keyword in text for keyword in ("browser", "website", "amazon", "google", "login", "shopping", "checkout", "orders")):
|
||||
return "Browser / web"
|
||||
if any(keyword in text for keyword in ("file", "folder", "directory", "terminal", "shell", "command", "cli", "script", "git", "repo", "install", "pip", "npm")):
|
||||
return "Files / terminal"
|
||||
if any(keyword in text for keyword in ("write", "summary", "document", "docs", "report", "email", "message", "readme", "markdown")):
|
||||
return "Writing / docs"
|
||||
if any(keyword in text for keyword in ("data", "analysis", "csv", "spreadsheet", "sheet", "table", "chart", "dashboard", "metric", "sql")):
|
||||
return "Data / analysis"
|
||||
if any(keyword in text for keyword in ("code", "bug", "fix", "test", "debug", "api", "backend", "frontend", "database", "deploy", "docker", "service", "build")):
|
||||
return "Development / ops"
|
||||
return "Other"
|
||||
|
||||
|
||||
class FakeJobManager:
|
||||
def __init__(self, *, config: AppConfig, db: Any, broadcast: Any = None) -> None:
|
||||
self.config = config
|
||||
@@ -94,8 +80,6 @@ class FakeJobManager:
|
||||
"started_at": created_at,
|
||||
"ended_at": None,
|
||||
"steps": 1,
|
||||
"result": "Running",
|
||||
"response": {"return": "Running", "data": None},
|
||||
"return": "Running",
|
||||
"data": None,
|
||||
"usage": {
|
||||
@@ -188,7 +172,6 @@ class FakeJobManager:
|
||||
|
||||
def analytics(self) -> dict[str, Any]:
|
||||
by_category: dict[str, dict[str, Any]] = {}
|
||||
by_day: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def bucket(target: dict[str, dict[str, Any]], key: str) -> dict[str, Any]:
|
||||
return target.setdefault(
|
||||
@@ -207,65 +190,28 @@ class FakeJobManager:
|
||||
},
|
||||
)
|
||||
|
||||
total_jobs = 0
|
||||
finished_jobs = 0
|
||||
completed_jobs = 0
|
||||
failed_jobs = 0
|
||||
cancelled_jobs = 0
|
||||
steps_sum = 0
|
||||
steps_count = 0
|
||||
cost_sum = 0.0
|
||||
cost_count = 0
|
||||
|
||||
for job in self._jobs.values():
|
||||
total_jobs += 1
|
||||
status = str(job.get("status") or "")
|
||||
finished = status in _TERMINAL_STATUSES
|
||||
category = _objective_category(str(job.get("objective") or ""))
|
||||
day = str(job.get("created_at") or "")[:10] or "unknown"
|
||||
|
||||
category_bucket = bucket(by_category, category)
|
||||
day_bucket = bucket(by_day, day)
|
||||
for item in (category_bucket, day_bucket):
|
||||
item["total_jobs"] += 1
|
||||
|
||||
category_bucket = bucket(by_category, _objective_category(str(job.get("objective") or "")))
|
||||
category_bucket["total_jobs"] += 1
|
||||
if not finished:
|
||||
continue
|
||||
|
||||
finished_jobs += 1
|
||||
category_bucket["finished_jobs"] += 1
|
||||
if status == "completed":
|
||||
completed_jobs += 1
|
||||
category_bucket["completed_jobs"] += 1
|
||||
elif status == "failed":
|
||||
failed_jobs += 1
|
||||
category_bucket["failed_jobs"] += 1
|
||||
elif status == "cancelled":
|
||||
cancelled_jobs += 1
|
||||
|
||||
category_bucket["cancelled_jobs"] += 1
|
||||
steps_raw = job.get("steps")
|
||||
if steps_raw is not None:
|
||||
steps = int(steps_raw)
|
||||
steps_sum += steps
|
||||
steps_count += 1
|
||||
for item in (category_bucket, day_bucket):
|
||||
item["steps_sum"] += steps
|
||||
item["steps_count"] += 1
|
||||
|
||||
category_bucket["steps_sum"] += int(steps_raw)
|
||||
category_bucket["steps_count"] += 1
|
||||
estimated_cost_raw = (job.get("usage") or {}).get("estimated_cost_usd")
|
||||
if estimated_cost_raw is not None:
|
||||
estimated_cost = float(estimated_cost_raw)
|
||||
cost_sum += estimated_cost
|
||||
cost_count += 1
|
||||
for item in (category_bucket, day_bucket):
|
||||
item["cost_sum"] += estimated_cost
|
||||
item["cost_count"] += 1
|
||||
|
||||
for item in (category_bucket, day_bucket):
|
||||
item["finished_jobs"] += 1
|
||||
if status == "completed":
|
||||
item["completed_jobs"] += 1
|
||||
elif status == "failed":
|
||||
item["failed_jobs"] += 1
|
||||
elif status == "cancelled":
|
||||
item["cancelled_jobs"] += 1
|
||||
category_bucket["cost_sum"] += float(estimated_cost_raw)
|
||||
category_bucket["cost_count"] += 1
|
||||
|
||||
def finalize(item: dict[str, Any]) -> dict[str, Any]:
|
||||
finished = item["finished_jobs"]
|
||||
@@ -281,18 +227,7 @@ class FakeJobManager:
|
||||
"avg_cost_usd": round(item["cost_sum"] / item["cost_count"], 6) if item["cost_count"] else None,
|
||||
}
|
||||
|
||||
return {
|
||||
"total_jobs": total_jobs,
|
||||
"finished_jobs": finished_jobs,
|
||||
"completed_jobs": completed_jobs,
|
||||
"failed_jobs": failed_jobs,
|
||||
"cancelled_jobs": cancelled_jobs,
|
||||
"success_rate": round((completed_jobs / finished_jobs) * 100, 2) if finished_jobs else 0.0,
|
||||
"avg_steps": round(steps_sum / steps_count, 2) if steps_count else None,
|
||||
"avg_cost_usd": round(cost_sum / cost_count, 6) if cost_count else None,
|
||||
"by_category": sorted((finalize(item) for item in by_category.values()), key=lambda item: (-item["success_rate"], item["label"])),
|
||||
"timeline": sorted((finalize(item) for item in by_day.values()), key=lambda item: item["label"]),
|
||||
}
|
||||
return {"by_category": sorted((finalize(item) for item in by_category.values()), key=lambda item: item["label"])}
|
||||
|
||||
|
||||
def _build_app(tmp_path: Path, monkeypatch: Any, disable_ui: bool = False):
|
||||
@@ -352,8 +287,8 @@ def test_create_job_returns_only_job_id_and_defaults_model(tmp_path: Path, monke
|
||||
status_res = client.get(f"/api/jobs/{job_id}/status", headers=headers)
|
||||
assert status_res.status_code == 200
|
||||
assert status_res.json()["job_id"] == job_id
|
||||
assert status_res.json()["response"]["return"] == "Running"
|
||||
assert "data" in status_res.json()["response"]
|
||||
assert status_res.json()["return"] == "Running"
|
||||
assert status_res.json()["data"] is None
|
||||
|
||||
|
||||
def test_create_job_rejects_invalid_disabled_tool_names(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
@@ -459,7 +394,7 @@ def test_replay_endpoint_skips_visual_paths_outside_artifacts(tmp_path: Path, mo
|
||||
assert payload["total_frames"] == 1
|
||||
|
||||
|
||||
def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
def test_analytics_endpoint_groups_by_category(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False)
|
||||
manager = app.state.manager
|
||||
client = TestClient(app)
|
||||
@@ -495,14 +430,6 @@ def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypa
|
||||
assert analytics.status_code == 200
|
||||
payload = analytics.json()
|
||||
|
||||
assert payload["total_jobs"] == 3
|
||||
assert payload["finished_jobs"] == 3
|
||||
assert payload["completed_jobs"] == 2
|
||||
assert payload["failed_jobs"] == 1
|
||||
assert payload["success_rate"] == 66.67
|
||||
assert payload["avg_steps"] == 6.67
|
||||
assert payload["avg_cost_usd"] == 0.136667
|
||||
|
||||
browser = next(row for row in payload["by_category"] if row["label"] == "Browser / web")
|
||||
terminal = next(row for row in payload["by_category"] if row["label"] == "Files / terminal")
|
||||
assert browser["finished_jobs"] == 2
|
||||
@@ -510,8 +437,6 @@ def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypa
|
||||
assert browser["avg_steps"] == 5.0
|
||||
assert terminal["success_rate"] == 100.0
|
||||
|
||||
assert [row["label"] for row in payload["timeline"]] == ["2026-05-27", "2026-05-28"]
|
||||
|
||||
|
||||
def test_ui_toggle(tmp_path: Path, monkeypatch: Any) -> None:
|
||||
app_enabled, _ = _build_app(tmp_path / "enabled", monkeypatch, disable_ui=False)
|
||||
|
||||
Reference in New Issue
Block a user