Add planner previews and streaming

skill/agent_runner.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Any, Dict, Sequence
+from typing import Any, Dict
 
 from .clickthrough_skill import ActionPlan, ClickthroughSkill
 
@@ -10,6 +10,7 @@ class AgentRunResult:
     action: Dict[str, Any]
     history: Dict[str, Any]
     grid: Dict[str, Any]
+    plan_preview: Dict[str, Any]
 
 
 class ClickthroughAgentRunner:
@@ -34,29 +35,26 @@ class ClickthroughAgentRunner:
             rows=rows,
             columns=columns,
         )
-        cells = grid.get("cells") or []
-        target_cell = self._choose_cell(cells, preferred_label)
-        plan = ActionPlan(
+        plan_response = self.skill.plan_with_planner(
             grid_id=grid["grid_id"],
-            target_cell=target_cell,
+            preferred_label=preferred_label,
             action=action,
             text=text,
         )
+        plan_payload = plan_response["plan"]
+        plan = ActionPlan(
+            grid_id=plan_payload["grid_id"],
+            target_cell=plan_payload.get("target_cell"),
+            action=plan_payload["action"],
+            text=plan_payload.get("text"),
+        )
         action_result = self.skill.plan_action(plan)
         summary = self.skill.grid_summary(grid["grid_id"])
         history = self.skill.grid_history(grid["grid_id"])
-        return AgentRunResult(summary=summary, action=action_result, history=history, grid=grid)
-
-    def _choose_cell(
-        self, cells: Sequence[dict[str, Any]], preferred_label: str | None
-    ) -> str:
-        if not cells:
-            raise ValueError("Grid contains no cells")
-        if preferred_label:
-            search = preferred_label.lower()
-            for cell in cells:
-                label_value = cell.get("label")
-                if label_value and search in label_value.lower():
-                    return cell["cell_id"]
-        center_index = len(cells) // 2
-        return cells[center_index]["cell_id"]
+        return AgentRunResult(
+            summary=summary,
+            action=action_result,
+            history=history,
+            grid=grid,
+            plan_preview=plan_response,
+        )

skill/clickthrough_skill.py

@@ -60,6 +60,30 @@ class ClickthroughSkill:
         response.raise_for_status()
         return response.json()
 
+    def plan_with_planner(
+        self,
+        grid_id: str,
+        preferred_label: str | None = None,
+        action: str = "click",
+        text: str | None = None,
+        comment: str | None = None,
+    ) -> Dict[str, Any]:
+        payload = {
+            "preferred_label": preferred_label,
+            "action": action,
+            "text": text,
+            "comment": comment or "planner-generated",
+        }
+        response = self._client.post(f"/grid/{grid_id}/plan", json=payload)
+        response.raise_for_status()
+        return response.json()
+
+    def refresh_grid(self, grid_id: str, screenshot_base64: str, memo: str | None = None) -> Dict[str, Any]:
+        payload = {"screenshot_base64": screenshot_base64, "memo": memo}
+        response = self._client.post(f"/grid/{grid_id}/refresh", json=payload)
+        response.raise_for_status()
+        return response.json()
+
 
 if __name__ == "__main__":
     import base64