From 5fa516f7e7aa2defdb78e1a9f68617abad409f66 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 5 Apr 2026 19:37:07 +0200 Subject: [PATCH] Add control UI --- client/app.js | 159 ++++++++++++++++++++++++++++++++++++++++++++++ client/index.html | 85 +++++++++++++++++++++++++ client/styles.css | 108 +++++++++++++++++++++++++++++++ server/main.py | 14 ++++ tests/test_ui.py | 12 ++++ 5 files changed, 378 insertions(+) create mode 100644 client/app.js create mode 100644 client/index.html create mode 100644 client/styles.css create mode 100644 tests/test_ui.py diff --git a/client/app.js b/client/app.js new file mode 100644 index 0000000..8fd6764 --- /dev/null +++ b/client/app.js @@ -0,0 +1,159 @@ +const gridForm = document.getElementById("grid-form"); +const descriptorEl = document.getElementById("descriptor"); +const gridMetaEl = document.getElementById("grid-meta"); +const summaryEl = document.getElementById("summary"); +const historyEl = document.getElementById("history"); +const planOutput = document.getElementById("plan-output"); +const preferredInput = document.getElementById("preferred-label"); +const refreshScreenshot = document.getElementById("refresh-screenshot"); +const refreshMemo = document.getElementById("refresh-memo"); +const logEl = document.getElementById("ws-log"); + +let currentGrid = null; +let lastPlan = null; +let ws = null; +let keepAliveId = null; + +const log = (message) => { + const timestamp = new Date().toLocaleTimeString(); + logEl.textContent = `[${timestamp}] ${message}\n${logEl.textContent}`; +}; + +const headers = { + "Content-Type": "application/json", +}; + +const subscribeToGrid = (gridId) => { + if (!gridId) return; + if (ws) { + ws.close(); + } + const protocol = window.location.protocol === "https:" ? "wss" : "ws"; + ws = new WebSocket(`${protocol}://${window.location.host}/stream/screenshots?grid_id=${gridId}`); + + ws.addEventListener("open", () => { + log(`WebSocket listening for grid ${gridId}`); + ws.send("ready"); + keepAliveId = setInterval(() => ws.send("ping"), 15000); + }); + + ws.addEventListener("message", (event) => { + log(`Update received → ${event.data}`); + }); + + ws.addEventListener("close", () => { + log("WebSocket disconnected"); + if (keepAliveId) { + clearInterval(keepAliveId); + keepAliveId = null; + } + }); +}; + +const updateDescriptor = (descriptor) => { + descriptorEl.textContent = JSON.stringify(descriptor, null, 2); + gridMetaEl.textContent = `Grid ${descriptor.grid_id} (${descriptor.rows}x${descriptor.columns}) · ${descriptor.cells.length} cells`; +}; + +const updateSummary = async () => { + if (!currentGrid) return; + const [summaryResponse, historyResponse] = await Promise.all([ + fetch(`/grid/${currentGrid}/summary`), + fetch(`/grid/${currentGrid}/history`), + ]); + + if (summaryResponse.ok) { + const payload = await summaryResponse.json(); + summaryEl.textContent = payload.summary; + } + + if (historyResponse.ok) { + const payload = await historyResponse.json(); + historyEl.textContent = JSON.stringify(payload.history, null, 2); + } +}; + +const initGrid = async (event) => { + event.preventDefault(); + const formData = new FormData(gridForm); + const payload = { + width: Number(formData.get("width")), + height: Number(formData.get("height")), + rows: Number(formData.get("rows")), + columns: Number(formData.get("columns")), + screenshot_base64: formData.get("screenshot"), + }; + const response = await fetch("/grid/init", { + method: "POST", + headers, + body: JSON.stringify(payload), + }); + const descriptor = await response.json(); + currentGrid = descriptor.grid_id; + updateDescriptor(descriptor); + await updateSummary(); + subscribeToGrid(currentGrid); + planOutput.textContent = "Plan preview will appear here."; + log(`Grid ${currentGrid} initialized.`); +}; + +document.getElementById("plan-button").addEventListener("click", async () => { + if (!currentGrid) { + log("Initialize a grid first."); + return; + } + const response = await fetch(`/grid/${currentGrid}/plan`, { + method: "POST", + headers, + body: JSON.stringify({ + preferred_label: preferredInput.value || null, + action: "click", + text: "ui-trigger", + }), + }); + const result = await response.json(); + lastPlan = result.plan; + planOutput.textContent = JSON.stringify(result, null, 2); +}); + +document.getElementById("run-action").addEventListener("click", async () => { + if (!lastPlan) { + log("Run the planner first."); + return; + } + const payload = { + grid_id: lastPlan.grid_id, + action: lastPlan.action, + target_cell: lastPlan.target_cell, + text: "from-ui", + comment: "UI action", + }; + const response = await fetch("/grid/action", { + method: "POST", + headers, + body: JSON.stringify(payload), + }); + const result = await response.json(); + log(`Action ${result.detail} at ${result.coordinates}`); + await updateSummary(); +}); + +document.getElementById("refresh-button").addEventListener("click", async () => { + if (!currentGrid) { + log("Start a grid first."); + return; + } + const payload = { + screenshot_base64: refreshScreenshot.value || "", + memo: refreshMemo.value || undefined, + }; + const response = await fetch(`/grid/${currentGrid}/refresh`, { + method: "POST", + headers, + body: JSON.stringify(payload), + }); + const data = await response.json(); + log(`Refresh acknowledged: ${JSON.stringify(data)}`); +}); + +gridForm.addEventListener("submit", initGrid); diff --git a/client/index.html b/client/index.html new file mode 100644 index 0000000..86206b9 --- /dev/null +++ b/client/index.html @@ -0,0 +1,85 @@ + + + + + Clickthrough Control + + + +
+
+

Clickthrough Control Panel

+

Most actions use HTTP; screenshots stream over WebSocket when refreshed.

+
+ +
+

Grid bootstrap

+
+ + + + + + +
+
+ +
+

Grid status

+
No grid yet.
+ 

+      

+
+      

+        
Planner & Actions

+        
+        

+          
+          
+        

+        
Plan preview will appear here.

+      

+
+      

+        
Refresh Screenshot

+        
+        
+        
+        
Refresh triggers /stream/screenshots so the UI can redraw.

+      

+
+      

+        
Summary & history

+        
No data yet.

+        
History will show here.

+      

+
+      

+        
Websocket log

+        
Waiting for updates…

+      

+    

+    
+  
+
diff --git a/client/styles.css b/client/styles.css
new file mode 100644
index 0000000..ec43300
--- /dev/null
+++ b/client/styles.css
@@ -0,0 +1,108 @@
+* {
+  box-sizing: border-box;
+}
+
+body {
+  font-family: "Inter", "Segoe UI", system-ui, sans-serif;
+  margin: 0;
+  background: #121212;
+  color: #f5f5f5;
+}
+
+main {
+  max-width: 960px;
+  margin: 0 auto;
+  padding: 24px;
+}
+
+header {
+  text-align: center;
+  margin-bottom: 24px;
+}
+
+header h1 {
+  margin-bottom: 8px;
+}
+
+.card {
+  background: #1f1f1f;
+  padding: 16px;
+  border-radius: 16px;
+  margin-bottom: 16px;
+  box-shadow: 0 20px 45px rgba(0, 0, 0, 0.35);
+}
+
+label {
+  display: block;
+  margin-bottom: 12px;
+}
+
+label input,
+label textarea {
+  width: 100%;
+  border-radius: 10px;
+  border: 1px solid #333;
+  background: #0f0f0f;
+  color: #f1f1f1;
+  padding: 8px 12px;
+  margin-top: 4px;
+  font-family: inherit;
+}
+
+textarea {
+  font-family: inherit;
+}
+
+button {
+  background: linear-gradient(135deg, #6d7cff, #3b82f6);
+  border: none;
+  padding: 10px 20px;
+  color: white;
+  border-radius: 999px;
+  font-weight: 600;
+  cursor: pointer;
+  transition: transform 0.15s ease;
+}
+
+button:hover {
+  transform: translateY(-1px);
+}
+
+.button-row {
+  display: flex;
+  gap: 12px;
+  flex-wrap: wrap;
+  margin-bottom: 12px;
+}
+
+.monospace {
+  background: #0c0c0c;
+  border-radius: 12px;
+  padding: 12px;
+  border: 1px solid #333;
+  min-height: 80px;
+}
+
+.note {
+  font-size: 0.9rem;
+  margin-top: 8px;
+  color: #b0b0b0;
+}
+
+@media (min-width: 768px) {
+  label {
+    display: flex;
+    gap: 12px;
+    align-items: center;
+  }
+
+  label input,
+  label textarea {
+    width: auto;
+    flex: 1;
+  }
+
+  .stretch textarea {
+    width: 100%;
+  }
+}
diff --git a/server/main.py b/server/main.py
index 5c2f62b..38baf55 100644
--- a/server/main.py
+++ b/server/main.py
@@ -1,6 +1,9 @@
 import time
+from pathlib import Path
 
 from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect
+from fastapi.responses import RedirectResponse
+from fastapi.staticfiles import StaticFiles
 
 from .config import ServerSettings
 from .grid import GridManager
@@ -26,6 +29,17 @@ app = FastAPI(
     version="0.3.0",
 )
 
+client_dir = Path(__file__).resolve().parent.parent / "client"
+if client_dir.exists():
+    app.mount("/ui", StaticFiles(directory=str(client_dir), html=True), name="ui")
+
+
+@app.get("/")
+async def root():
+    if client_dir.exists():
+        return RedirectResponse("/ui/")
+    return {"status": "ok", "grid_count": manager.grid_count}
+
 
 @app.get("/health")
 def health_check() -> dict[str, str]:
diff --git a/tests/test_ui.py b/tests/test_ui.py
new file mode 100644
index 0000000..e3222ba
--- /dev/null
+++ b/tests/test_ui.py
@@ -0,0 +1,12 @@
+from fastapi.testclient import TestClient
+
+from server.main import app
+
+
+test_client = TestClient(app)
+
+
+def test_ui_root_serves_index():
+    response = test_client.get("/ui/")
+    assert response.status_code == 200
+    assert "Clickthrough Control" in response.text