Add control UI

client/app.js

@@ -0,0 +1,159 @@
+const gridForm = document.getElementById("grid-form");
+const descriptorEl = document.getElementById("descriptor");
+const gridMetaEl = document.getElementById("grid-meta");
+const summaryEl = document.getElementById("summary");
+const historyEl = document.getElementById("history");
+const planOutput = document.getElementById("plan-output");
+const preferredInput = document.getElementById("preferred-label");
+const refreshScreenshot = document.getElementById("refresh-screenshot");
+const refreshMemo = document.getElementById("refresh-memo");
+const logEl = document.getElementById("ws-log");
+
+let currentGrid = null;
+let lastPlan = null;
+let ws = null;
+let keepAliveId = null;
+
+const log = (message) => {
+  const timestamp = new Date().toLocaleTimeString();
+  logEl.textContent = `[${timestamp}] ${message}\n${logEl.textContent}`;
+};
+
+const headers = {
+  "Content-Type": "application/json",
+};
+
+const subscribeToGrid = (gridId) => {
+  if (!gridId) return;
+  if (ws) {
+    ws.close();
+  }
+  const protocol = window.location.protocol === "https:" ? "wss" : "ws";
+  ws = new WebSocket(`${protocol}://${window.location.host}/stream/screenshots?grid_id=${gridId}`);
+
+  ws.addEventListener("open", () => {
+    log(`WebSocket listening for grid ${gridId}`);
+    ws.send("ready");
+    keepAliveId = setInterval(() => ws.send("ping"), 15000);
+  });
+
+  ws.addEventListener("message", (event) => {
+    log(`Update received → ${event.data}`);
+  });
+
+  ws.addEventListener("close", () => {
+    log("WebSocket disconnected");
+    if (keepAliveId) {
+      clearInterval(keepAliveId);
+      keepAliveId = null;
+    }
+  });
+};
+
+const updateDescriptor = (descriptor) => {
+  descriptorEl.textContent = JSON.stringify(descriptor, null, 2);
+  gridMetaEl.textContent = `Grid ${descriptor.grid_id} (${descriptor.rows}x${descriptor.columns}) · ${descriptor.cells.length} cells`;
+};
+
+const updateSummary = async () => {
+  if (!currentGrid) return;
+  const [summaryResponse, historyResponse] = await Promise.all([
+    fetch(`/grid/${currentGrid}/summary`),
+    fetch(`/grid/${currentGrid}/history`),
+  ]);
+
+  if (summaryResponse.ok) {
+    const payload = await summaryResponse.json();
+    summaryEl.textContent = payload.summary;
+  }
+
+  if (historyResponse.ok) {
+    const payload = await historyResponse.json();
+    historyEl.textContent = JSON.stringify(payload.history, null, 2);
+  }
+};
+
+const initGrid = async (event) => {
+  event.preventDefault();
+  const formData = new FormData(gridForm);
+  const payload = {
+    width: Number(formData.get("width")),
+    height: Number(formData.get("height")),
+    rows: Number(formData.get("rows")),
+    columns: Number(formData.get("columns")),
+    screenshot_base64: formData.get("screenshot"),
+  };
+  const response = await fetch("/grid/init", {
+    method: "POST",
+    headers,
+    body: JSON.stringify(payload),
+  });
+  const descriptor = await response.json();
+  currentGrid = descriptor.grid_id;
+  updateDescriptor(descriptor);
+  await updateSummary();
+  subscribeToGrid(currentGrid);
+  planOutput.textContent = "Plan preview will appear here.";
+  log(`Grid ${currentGrid} initialized.`);
+};
+
+document.getElementById("plan-button").addEventListener("click", async () => {
+  if (!currentGrid) {
+    log("Initialize a grid first.");
+    return;
+  }
+  const response = await fetch(`/grid/${currentGrid}/plan`, {
+    method: "POST",
+    headers,
+    body: JSON.stringify({
+      preferred_label: preferredInput.value || null,
+      action: "click",
+      text: "ui-trigger",
+    }),
+  });
+  const result = await response.json();
+  lastPlan = result.plan;
+  planOutput.textContent = JSON.stringify(result, null, 2);
+});
+
+document.getElementById("run-action").addEventListener("click", async () => {
+  if (!lastPlan) {
+    log("Run the planner first.");
+    return;
+  }
+  const payload = {
+    grid_id: lastPlan.grid_id,
+    action: lastPlan.action,
+    target_cell: lastPlan.target_cell,
+    text: "from-ui",
+    comment: "UI action",
+  };
+  const response = await fetch("/grid/action", {
+    method: "POST",
+    headers,
+    body: JSON.stringify(payload),
+  });
+  const result = await response.json();
+  log(`Action ${result.detail} at ${result.coordinates}`);
+  await updateSummary();
+});
+
+document.getElementById("refresh-button").addEventListener("click", async () => {
+  if (!currentGrid) {
+    log("Start a grid first.");
+    return;
+  }
+  const payload = {
+    screenshot_base64: refreshScreenshot.value || "",
+    memo: refreshMemo.value || undefined,
+  };
+  const response = await fetch(`/grid/${currentGrid}/refresh`, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(payload),
+  });
+  const data = await response.json();
+  log(`Refresh acknowledged: ${JSON.stringify(data)}`);
+});
+
+gridForm.addEventListener("submit", initGrid);

client/index.html

@@ -0,0 +1,85 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>Clickthrough Control</title>
+    <link rel="stylesheet" href="styles.css" />
+  </head>
+  <body>
+    <main>
+      <header>
+        <h1>Clickthrough Control Panel</h1>
+        <p>Most actions use HTTP; screenshots stream over WebSocket when refreshed.</p>
+      </header>
+
+      <section class="card">
+        <h2>Grid bootstrap</h2>
+        <form id="grid-form">
+          <label>
+            Width
+            <input type="number" name="width" value="640" min="1" required />
+          </label>
+          <label>
+            Height
+            <input type="number" name="height" value="480" min="1" required />
+          </label>
+          <label>
+            Rows
+            <input type="number" name="rows" value="3" min="1" required />
+          </label>
+          <label>
+            Columns
+            <input type="number" name="columns" value="3" min="1" required />
+          </label>
+          <label class="stretch">
+            Screenshot (base64)
+            <textarea name="screenshot" id="screenshot" rows="3">AA==</textarea>
+          </label>
+          <button type="submit">Init grid</button>
+        </form>
+      </section>
+
+      <section class="card" id="grid-details">
+        <h2>Grid status</h2>
+        <div id="grid-meta">No grid yet.</div>
+        <pre id="descriptor" class="monospace"></pre>
+      </section>
+
+      <section class="card">
+        <h2>Planner &amp; Actions</h2>
+        <label>
+          Preferred label
+          <input type="text" id="preferred-label" placeholder="button" />
+        </label>
+        <div class="button-row">
+          <button type="button" id="plan-button">Preview plan</button>
+          <button type="button" id="run-action">Run action</button>
+        </div>
+        <pre id="plan-output" class="monospace">Plan preview will appear here.</pre>
+      </section>
+
+      <section class="card">
+        <h2>Refresh Screenshot</h2>
+        <textarea id="refresh-screenshot" rows="3" placeholder="Paste new base64 screenshot"></textarea>
+        <label>
+          Memo
+          <input type="text" id="refresh-memo" placeholder="Describe the scene" />
+        </label>
+        <button type="button" id="refresh-button">Refresh grid</button>
+        <p class="note">Refresh triggers /stream/screenshots so the UI can redraw.</p>
+      </section>
+
+      <section class="card">
+        <h2>Summary &amp; history</h2>
+        <pre id="summary" class="monospace">No data yet.</pre>
+        <pre id="history" class="monospace">History will show here.</pre>
+      </section>
+
+      <section class="card">
+        <h2>Websocket log</h2>
+        <pre id="ws-log" class="monospace">Waiting for updates…</pre>
+      </section>
+    </main>
+    <script type="module" src="app.js"></script>
+  </body>
+</html>

client/styles.css

@@ -0,0 +1,108 @@
+* {
+  box-sizing: border-box;
+}
+
+body {
+  font-family: "Inter", "Segoe UI", system-ui, sans-serif;
+  margin: 0;
+  background: #121212;
+  color: #f5f5f5;
+}
+
+main {
+  max-width: 960px;
+  margin: 0 auto;
+  padding: 24px;
+}
+
+header {
+  text-align: center;
+  margin-bottom: 24px;
+}
+
+header h1 {
+  margin-bottom: 8px;
+}
+
+.card {
+  background: #1f1f1f;
+  padding: 16px;
+  border-radius: 16px;
+  margin-bottom: 16px;
+  box-shadow: 0 20px 45px rgba(0, 0, 0, 0.35);
+}
+
+label {
+  display: block;
+  margin-bottom: 12px;
+}
+
+label input,
+label textarea {
+  width: 100%;
+  border-radius: 10px;
+  border: 1px solid #333;
+  background: #0f0f0f;
+  color: #f1f1f1;
+  padding: 8px 12px;
+  margin-top: 4px;
+  font-family: inherit;
+}
+
+textarea {
+  font-family: inherit;
+}
+
+button {
+  background: linear-gradient(135deg, #6d7cff, #3b82f6);
+  border: none;
+  padding: 10px 20px;
+  color: white;
+  border-radius: 999px;
+  font-weight: 600;
+  cursor: pointer;
+  transition: transform 0.15s ease;
+}
+
+button:hover {
+  transform: translateY(-1px);
+}
+
+.button-row {
+  display: flex;
+  gap: 12px;
+  flex-wrap: wrap;
+  margin-bottom: 12px;
+}
+
+.monospace {
+  background: #0c0c0c;
+  border-radius: 12px;
+  padding: 12px;
+  border: 1px solid #333;
+  min-height: 80px;
+}
+
+.note {
+  font-size: 0.9rem;
+  margin-top: 8px;
+  color: #b0b0b0;
+}
+
+@media (min-width: 768px) {
+  label {
+    display: flex;
+    gap: 12px;
+    align-items: center;
+  }
+
+  label input,
+  label textarea {
+    width: auto;
+    flex: 1;
+  }
+
+  .stretch textarea {
+    width: 100%;
+  }
+}

server/main.py

@@ -1,6 +1,9 @@
 import time
+from pathlib import Path
 
 from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
 
 from .config import ServerSettings
 from .grid import GridManager
@@ -26,6 +29,17 @@ app = FastAPI(
     version="0.3.0",
 )
 
+client_dir = Path(__file__).resolve().parent.parent / "client"
+if client_dir.exists():
+    app.mount("/ui", StaticFiles(directory=str(client_dir), html=True), name="ui")
+
+
+@app.get("/")
+async def root():
+    if client_dir.exists():
+        return RedirectResponse("/ui/")
+    return {"status": "ok", "grid_count": manager.grid_count}
+
 
 @app.get("/health")
 def health_check() -> dict[str, str]:

tests/test_ui.py

@@ -0,0 +1,12 @@
+from fastapi.testclient import TestClient
+
+from server.main import app
+
+
+test_client = TestClient(app)
+
+
+def test_ui_root_serves_index():
+    response = test_client.get("/ui/")
+    assert response.status_code == 200
+    assert "Clickthrough Control" in response.text