From 75a2600e4d2437abd941f2b0942d4ba5197d6110 Mon Sep 17 00:00:00 2001 From: space Date: Thu, 4 Jun 2026 18:00:26 +0200 Subject: [PATCH] feat. Revision of Agent Lopp & major clean-up --- .pi/skills/death-is-soon-wrap-up/SKILL.md | 23 + README.md | 64 +- install_tray_startup_shortcut.ps1 | 47 -- screenjob_tray.ps1 | 307 ---------- .../BackendProcessService.cs | 138 ----- .../ScreenJob.WindowsServiceHost/Program.cs | 18 - .../ScreenJob.WindowsServiceHost.csproj | 12 - .../ServiceOptions.cs | 77 --- src/agent.py | 556 ++++++++++-------- src/app_main.py | 1 + src/cli.py | 102 ++-- src/config.py | 6 + src/desktop_overlay.py | 286 --------- src/runtime.py | 40 +- src/server.py | 16 +- src/storage.py | 89 ++- src/task_manager.py | 51 +- src/ui_assets/monitoring.html | 20 +- src/ui_assets/monitoring.js | 117 +--- src/utils.py | 43 +- start_screenjob_tray_hidden.vbs | 11 - tests/test_agent_tools.py | 178 +++--- tests/test_cli.py | 22 - tests/test_desktop_overlay.py | 181 ------ tests/test_server_api.py | 105 +--- tests/test_storage.py | 46 +- tests/test_task_manager.py | 85 +-- tray_service_control.ps1 | 53 -- 28 files changed, 753 insertions(+), 1941 deletions(-) create mode 100644 .pi/skills/death-is-soon-wrap-up/SKILL.md delete mode 100644 install_tray_startup_shortcut.ps1 delete mode 100644 screenjob_tray.ps1 delete mode 100644 service_host/ScreenJob.WindowsServiceHost/BackendProcessService.cs delete mode 100644 service_host/ScreenJob.WindowsServiceHost/Program.cs delete mode 100644 service_host/ScreenJob.WindowsServiceHost/ScreenJob.WindowsServiceHost.csproj delete mode 100644 service_host/ScreenJob.WindowsServiceHost/ServiceOptions.cs delete mode 100644 src/desktop_overlay.py delete mode 100644 start_screenjob_tray_hidden.vbs delete mode 100644 tests/test_desktop_overlay.py delete mode 100644 tray_service_control.ps1 diff --git a/.pi/skills/death-is-soon-wrap-up/SKILL.md b/.pi/skills/death-is-soon-wrap-up/SKILL.md new file mode 100644 index 0000000..a298b9a --- /dev/null +++ b/.pi/skills/death-is-soon-wrap-up/SKILL.md @@ -0,0 +1,23 @@ +--- +name: death-is-soon-wrap-up +description: Use when the user says "death is soon, wrap up" or wants the conversation closed out with a final note, concise summary, goodbye, and instruction to reset the chat. +--- + +# Death Is Soon Wrap Up + +## Purpose + +Use this skill to close a conversation cleanly and intentionally. + +## Wrap-Up Flow + +1. Note down anything that has a place to live. +1. If there are notes, tasks, summaries, or memory slots available, capture the important details there before ending. +1. Give the user one last short summary of the situation, decisions, and any next steps. +1. Say goodbye plainly. +1. Tell the user to reset the chat. + +## Output Style + +Keep the closing message short and calm. +Do not reopen the discussion after the goodbye unless the user asks for more. diff --git a/README.md b/README.md index a00506c..1a281b1 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,12 @@ It lets an LLM use controlled local tools (screen, mouse, keyboard, clipboard, s - Returns structured agent output as: - `return`: human-readable completion message - `data`: structured payload (for example command output) +- Cleans up old runs and history on startup (default retention: 7 days) ## Core Features - Hybrid control model: screenshot grounding plus Windows-native window, dialog, and UI-element helpers when available -- Tool-based agent loop (`execute_command`, `see_screen`, `enhance`, `list_windows`, `find_window`, `focus_window`, `close_window`, `wait_for_window`, `wait_for_focus_change`, `detect_dialog`, `dialog_action`, `dialog_set_filename`, `wait_for_dialog_close`, `list_ui_elements`, `invoke_ui_element`, `set_ui_element_value`, `select_ui_element`, `wait_for_ui_element`, `click`, `scroll`, `drag`, `move_mouse`, `type`, `press_key`, `clipboard_get`, `clipboard_set`, `get_cursor_position`, `get_active_window`, `sleep`, `task_complete`) +- Tool-based agent loop (`execute_command`, `enhance`, `list_windows`, `find_window`, `focus_window`, `close_window`, `wait_for_window`, `wait_for_focus_change`, `detect_dialog`, `dialog_action`, `dialog_set_filename`, `wait_for_dialog_close`, `list_ui_elements`, `invoke_ui_element`, `set_ui_element_value`, `select_ui_element`, `wait_for_ui_element`, `click`, `scroll`, `drag`, `move_mouse`, `type`, `press_key`, `clipboard_get`, `clipboard_set`, `get_cursor_position`, `sleep`, `task_complete`) - Safety pre-check with override support - Per-job tool disable list - Live/final usage and cost estimates @@ -73,6 +74,7 @@ SCREENJOB_SAFETY_MODEL=gpt-5.4-mini SCREENJOB_HOST=127.0.0.1 SCREENJOB_PORT=8787 DISABLE_UI=false +SCREENJOB_RETENTION_DAYS=7 ``` ## Usage @@ -83,16 +85,11 @@ DISABLE_UI=false python main.py run "Open amazon.de and go to my orders" ``` -CLI JSON output includes both legacy and structured fields: +CLI JSON output: ```json { "completed": true, - "result": "Task completed successfully", - "response": { - "return": "Task completed successfully", - "data": "file1.txt\nfile2.txt" - }, "return": "Task completed successfully", "data": "file1.txt\nfile2.txt" } @@ -147,43 +144,6 @@ If you need to start the backend manually, run: .\start_backend.ps1 ``` -The legacy Windows service host remains in the tree for reference, but it is not the recommended path for GUI tasks. - -### System Tray Icon (Windows) - -Start tray icon now: - -```powershell -powershell -NoProfile -ExecutionPolicy Bypass -STA -File .\screenjob_tray.ps1 -``` - -Install startup shortcut (current user): - -```powershell -.\install_tray_startup_shortcut.ps1 -``` - -Install startup shortcut for all users: - -```powershell -.\install_tray_startup_shortcut.ps1 -AllUsers -``` - -Remove startup shortcut: - -```powershell -.\install_tray_startup_shortcut.ps1 -Remove -``` - -Tray menu actions: - -- The service controls are for the legacy Windows service host. -- Refresh service status -- Start/Stop/Restart service (prompts for admin/UAC) -- Open dashboard URL from `.env` `SCREENJOB_HOST` / `SCREENJOB_PORT` -- Open service logs folder -- Exit tray icon process - Auth for all API routes: - `Authorization: Bearer ` @@ -225,10 +185,8 @@ Response: Each job payload includes: -- `result` (compat string) -- `response.return` -- `response.data` -- top-level `return` and `data` aliases +- `return` +- `data` ### Monitoring UI @@ -236,20 +194,20 @@ Each job payload includes: - Read-only dashboard (no run controls) - Requires token input - Live updates via `/ws` -- Analytics dashboards for success rate by objective category and daily averages +- Analytics dashboard for success by objective category - Set `DISABLE_UI=true` to disable UI ### Analytics API - `GET /api/analytics` -- Returns objective-category success rates plus average steps/cost over time +- Returns objective-category success rates plus average steps/cost by category ## Agent Instructions (Practical) - Prefer `execute_command` for deterministic actions (opening URLs, filesystem checks). - First classify the current Windows surface, then choose the control channel. - Prefer native window/dialog/element tools for focus changes, file pickers, modal confirmations, and browser-owned dialogs when available. -- Use `see_screen` before UI interaction. +- Use `enhance` for screen observation. Call it with no coordinate for a full-screen grid capture. - Use `enhance` before clicking small/ambiguous targets; prefer `region="small"` for compact controls. - Use `enhance` `mode="text"` for tiny labels/text, or `mode="ui"` for general UI. - Optionally set `enhance` `scale` (2-6) for tighter zoom control. @@ -261,11 +219,11 @@ Each job payload includes: - Use `click` offsets via `offset_up/down/left/right`; set `button` and `click_count` there instead of inventing one-off click tools. - Use `move_mouse` when you need hover-only behavior and `drag` for slider, selection, or window moves. - Use `scroll` for vertical navigation; positive amounts scroll up and negative amounts scroll down. -- Use `clipboard_get` / `clipboard_set` for copy-paste workflows, `get_cursor_position` for cursor inspection, and `get_active_window` before interacting with uncertain focus. +- Use `clipboard_get` / `clipboard_set` for copy-paste workflows, `get_cursor_position` for cursor inspection, and rely on the injected foreground-window context before interacting with uncertain focus. - If native automation is unavailable or disabled, ScreenJob falls back to screenshots plus mouse/keyboard control and emits fallback events. - When done, call: - `task_complete(return="...", data=...)` -- Before `task_complete`, verify expected on-screen content with `see_screen` (and `enhance` if needed), and include an `observed_result` summary in `data`. +- Before `task_complete`, verify expected on-screen content with the latest retained screen and `enhance` if needed, and include an `observed_result` summary in `data`. Per-job `disabled_tools` must match the built-in tool allowlist. `task_complete` cannot be disabled. diff --git a/install_tray_startup_shortcut.ps1 b/install_tray_startup_shortcut.ps1 deleted file mode 100644 index eb87cb9..0000000 --- a/install_tray_startup_shortcut.ps1 +++ /dev/null @@ -1,47 +0,0 @@ -[CmdletBinding(SupportsShouldProcess = $true)] -param( - [switch]$Remove, - [switch]$AllUsers -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -$scriptDir = Split-Path -Parent $PSCommandPath -$vbsLauncher = Join-Path $scriptDir "start_screenjob_tray_hidden.vbs" -$shortcutName = "ScreenJob Tray.lnk" - -if (-not (Test-Path -LiteralPath $vbsLauncher)) { - throw "Launcher file not found: $vbsLauncher" -} - -$startupFolder = if ($AllUsers) { - [Environment]::GetFolderPath("CommonStartup") -} else { - [Environment]::GetFolderPath("Startup") -} - -$shortcutPath = Join-Path $startupFolder $shortcutName - -if ($Remove) { - if (Test-Path -LiteralPath $shortcutPath) { - if ($PSCmdlet.ShouldProcess($shortcutPath, "Remove startup shortcut")) { - Remove-Item -LiteralPath $shortcutPath -Force - Write-Host "Removed startup shortcut: $shortcutPath" - } - } else { - Write-Host "No startup shortcut found at: $shortcutPath" - } - return -} - -if ($PSCmdlet.ShouldProcess($shortcutPath, "Create startup shortcut")) { - $shell = New-Object -ComObject WScript.Shell - $shortcut = $shell.CreateShortcut($shortcutPath) - $shortcut.TargetPath = "$env:SystemRoot\System32\wscript.exe" - $shortcut.Arguments = '"' + $vbsLauncher + '"' - $shortcut.WorkingDirectory = $scriptDir - $shortcut.Description = "Launch ScreenJob tray icon at sign-in." - $shortcut.Save() - Write-Host "Created startup shortcut: $shortcutPath" -} diff --git a/screenjob_tray.ps1 b/screenjob_tray.ps1 deleted file mode 100644 index 609c1ef..0000000 --- a/screenjob_tray.ps1 +++ /dev/null @@ -1,307 +0,0 @@ -param( - [string]$ServiceName = "ScreenJobBackend" -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -Add-Type -AssemblyName System.Windows.Forms -Add-Type -AssemblyName System.Drawing - -$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$controlScript = Join-Path $scriptDir "tray_service_control.ps1" -$logsDir = Join-Path $scriptDir "screenjob_runs\service" -$defaultHost = "127.0.0.1" -$defaultPort = "8787" - -function Read-EnvConfig { - param([string]$EnvFilePath) - $result = @{} - if (-not (Test-Path -LiteralPath $EnvFilePath)) { - return $result - } - - foreach ($line in Get-Content -Path $EnvFilePath) { - $trimmed = $line.Trim() - if ($trimmed.Length -eq 0 -or $trimmed.StartsWith("#")) { - continue - } - $parts = $trimmed.Split("=", 2) - if ($parts.Count -eq 2) { - $key = $parts[0].Trim() - $value = $parts[1].Trim() - if (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'"))) { - $value = $value.Substring(1, $value.Length - 2) - } - $result[$key] = $value - } - } - return $result -} - -function Get-ServiceStatusSafe { - param([string]$Name) - try { - $svc = Get-Service -Name $Name -ErrorAction Stop - return $svc.Status.ToString() - } catch { - return "NotInstalled" - } -} - -function Invoke-ServiceActionElevated { - param( - [Parameter(Mandatory = $true)][string]$Action, - [Parameter(Mandatory = $true)][string]$Name - ) - - if (-not (Test-Path -LiteralPath $controlScript)) { - [System.Windows.Forms.MessageBox]::Show( - "Missing control script: $controlScript", - "ScreenJob Tray", - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Error - ) | Out-Null - return - } - - $argList = @( - "-NoProfile", - "-ExecutionPolicy", "Bypass", - "-File", "`"$controlScript`"", - "-Action", $Action, - "-ServiceName", $Name - ) - - try { - Start-Process -FilePath "powershell.exe" -ArgumentList $argList -Verb RunAs -WindowStyle Hidden | Out-Null - } catch { - # User canceled UAC prompt or launch failed. - } -} - -function Get-DashboardUrl { - $envFile = Join-Path $scriptDir ".env" - $envVars = Read-EnvConfig -EnvFilePath $envFile - - $dashboardHost = $defaultHost - $dashboardPort = $defaultPort - - if ($envVars.ContainsKey("SCREENJOB_HOST") -and -not [string]::IsNullOrWhiteSpace($envVars["SCREENJOB_HOST"])) { - $dashboardHost = $envVars["SCREENJOB_HOST"] - } - if ($envVars.ContainsKey("SCREENJOB_PORT") -and -not [string]::IsNullOrWhiteSpace($envVars["SCREENJOB_PORT"])) { - $dashboardPort = $envVars["SCREENJOB_PORT"] - } - - $connectHost = Resolve-ConnectHost -ConfiguredHost $dashboardHost - return "http://{0}:{1}/" -f $connectHost, $dashboardPort -} - -function Resolve-ConnectHost { - param([string]$ConfiguredHost) - - if ([string]::IsNullOrWhiteSpace($ConfiguredHost)) { - return "127.0.0.1" - } - - switch ($ConfiguredHost.Trim().ToLowerInvariant()) { - "0.0.0.0" { return "127.0.0.1" } - "::" { return "127.0.0.1" } - "*" { return "127.0.0.1" } - default { return $ConfiguredHost } - } -} - -function Get-HealthCheckHosts { - param([string]$ConfiguredHost) - - if ([string]::IsNullOrWhiteSpace($ConfiguredHost)) { - return @("127.0.0.1", "localhost") - } - - $normalized = $ConfiguredHost.Trim().ToLowerInvariant() - switch ($normalized) { - "0.0.0.0" { return @("127.0.0.1", "localhost", "::1") } - "::" { return @("127.0.0.1", "localhost", "::1") } - "*" { return @("127.0.0.1", "localhost", "::1") } - default { return @($ConfiguredHost) } - } -} - -function Test-TcpEndpoint { - param( - [Parameter(Mandatory = $true)][string]$HostName, - [Parameter(Mandatory = $true)][int]$Port, - [int]$TimeoutMs = 1200 - ) - - $client = New-Object System.Net.Sockets.TcpClient - try { - $async = $client.BeginConnect($HostName, $Port, $null, $null) - $connected = $async.AsyncWaitHandle.WaitOne($TimeoutMs, $false) - if (-not $connected) { - return $false - } - $client.EndConnect($async) | Out-Null - return $true - } catch { - return $false - } finally { - $client.Dispose() - } -} - -function Get-BackendReachability { - $envFile = Join-Path $scriptDir ".env" - $envVars = Read-EnvConfig -EnvFilePath $envFile - $configuredHost = $defaultHost - $configuredPort = $defaultPort - - if ($envVars.ContainsKey("SCREENJOB_HOST") -and -not [string]::IsNullOrWhiteSpace($envVars["SCREENJOB_HOST"])) { - $configuredHost = $envVars["SCREENJOB_HOST"] - } - if ($envVars.ContainsKey("SCREENJOB_PORT") -and -not [string]::IsNullOrWhiteSpace($envVars["SCREENJOB_PORT"])) { - $configuredPort = $envVars["SCREENJOB_PORT"] - } - - $portNumber = 8787 - [void][int]::TryParse([string]$configuredPort, [ref]$portNumber) - $hostsToTry = Get-HealthCheckHosts -ConfiguredHost $configuredHost - - foreach ($candidateHost in $hostsToTry) { - if (Test-TcpEndpoint -HostName $candidateHost -Port $portNumber) { - return $true - } - } - - return $false -} - -function Update-TrayState { - param( - [System.Windows.Forms.NotifyIcon]$NotifyIcon, - [System.Windows.Forms.ToolStripMenuItem]$StatusItem, - [string]$Name - ) - - $status = Get-ServiceStatusSafe -Name $Name - $isBackendReachable = Get-BackendReachability - - $displayStatus = $status - if ($status -eq "Running" -and -not $isBackendReachable) { - $displayStatus = "Running (Backend Down)" - } elseif ($status -eq "Stopped" -and $isBackendReachable) { - $displayStatus = "Stopped (Backend Up)" - } elseif ($status -eq "NotInstalled" -and $isBackendReachable) { - $displayStatus = "NotInstalled (Backend Up)" - } - - $StatusItem.Text = "Status: $displayStatus" - - switch ($displayStatus) { - "Running" { - $NotifyIcon.Icon = [System.Drawing.SystemIcons]::Information - } - "Stopped" { - $NotifyIcon.Icon = [System.Drawing.SystemIcons]::Warning - } - default { - $NotifyIcon.Icon = [System.Drawing.SystemIcons]::Error - } - } - - $tooltip = "ScreenJob Backend: $displayStatus" - if ($tooltip.Length -gt 63) { - $tooltip = $tooltip.Substring(0, 63) - } - $NotifyIcon.Text = $tooltip -} - -$appContext = New-Object System.Windows.Forms.ApplicationContext -$notifyIcon = New-Object System.Windows.Forms.NotifyIcon -$notifyIcon.Visible = $false - -$menu = New-Object System.Windows.Forms.ContextMenuStrip -$statusItem = New-Object System.Windows.Forms.ToolStripMenuItem "Status: Unknown" -$statusItem.Enabled = $false - -$refreshItem = New-Object System.Windows.Forms.ToolStripMenuItem "Refresh Status" -$refreshItem.Add_Click({ - Update-TrayState -NotifyIcon $notifyIcon -StatusItem $statusItem -Name $ServiceName -}) - -$startItem = New-Object System.Windows.Forms.ToolStripMenuItem "Start Service (Admin)" -$startItem.Add_Click({ - Invoke-ServiceActionElevated -Action "start" -Name $ServiceName -}) - -$stopItem = New-Object System.Windows.Forms.ToolStripMenuItem "Stop Service (Admin)" -$stopItem.Add_Click({ - Invoke-ServiceActionElevated -Action "stop" -Name $ServiceName -}) - -$restartItem = New-Object System.Windows.Forms.ToolStripMenuItem "Restart Service (Admin)" -$restartItem.Add_Click({ - Invoke-ServiceActionElevated -Action "restart" -Name $ServiceName -}) - -$dashboardItem = New-Object System.Windows.Forms.ToolStripMenuItem "Open Dashboard" -$dashboardItem.Add_Click({ - $url = Get-DashboardUrl - Start-Process $url | Out-Null -}) - -$logsItem = New-Object System.Windows.Forms.ToolStripMenuItem "Open Service Logs" -$logsItem.Add_Click({ - if (-not (Test-Path -LiteralPath $logsDir)) { - New-Item -ItemType Directory -Path $logsDir -Force | Out-Null - } - Start-Process explorer.exe $logsDir | Out-Null -}) - -$openFolderItem = New-Object System.Windows.Forms.ToolStripMenuItem "Open Project Folder" -$openFolderItem.Add_Click({ - Start-Process explorer.exe $scriptDir | Out-Null -}) - -$exitItem = New-Object System.Windows.Forms.ToolStripMenuItem "Exit Tray" -$exitItem.Add_Click({ - $refreshTimer.Stop() - $notifyIcon.Visible = $false - $notifyIcon.Dispose() - $menu.Dispose() - $appContext.ExitThread() -}) - -[void]$menu.Items.Add($statusItem) -[void]$menu.Items.Add($refreshItem) -[void]$menu.Items.Add((New-Object System.Windows.Forms.ToolStripSeparator)) -[void]$menu.Items.Add($startItem) -[void]$menu.Items.Add($stopItem) -[void]$menu.Items.Add($restartItem) -[void]$menu.Items.Add((New-Object System.Windows.Forms.ToolStripSeparator)) -[void]$menu.Items.Add($dashboardItem) -[void]$menu.Items.Add($logsItem) -[void]$menu.Items.Add($openFolderItem) -[void]$menu.Items.Add((New-Object System.Windows.Forms.ToolStripSeparator)) -[void]$menu.Items.Add($exitItem) - -$notifyIcon.ContextMenuStrip = $menu -$notifyIcon.Visible = $true - -$notifyIcon.Add_DoubleClick({ - $url = Get-DashboardUrl - Start-Process $url | Out-Null -}) - -$refreshTimer = New-Object System.Windows.Forms.Timer -$refreshTimer.Interval = 5000 -$refreshTimer.Add_Tick({ - Update-TrayState -NotifyIcon $notifyIcon -StatusItem $statusItem -Name $ServiceName -}) - -Update-TrayState -NotifyIcon $notifyIcon -StatusItem $statusItem -Name $ServiceName -$refreshTimer.Start() - -[System.Windows.Forms.Application]::Run($appContext) diff --git a/service_host/ScreenJob.WindowsServiceHost/BackendProcessService.cs b/service_host/ScreenJob.WindowsServiceHost/BackendProcessService.cs deleted file mode 100644 index 01492f7..0000000 --- a/service_host/ScreenJob.WindowsServiceHost/BackendProcessService.cs +++ /dev/null @@ -1,138 +0,0 @@ -using System.Diagnostics; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -namespace ScreenJob.WindowsServiceHost; - -internal sealed class BackendProcessService : BackgroundService -{ - private readonly ILogger _logger; - private readonly ServiceOptions _options; - private readonly object _logLock = new(); - - private Process? _backendProcess; - private string _stdoutLogPath = string.Empty; - private string _stderrLogPath = string.Empty; - - public BackendProcessService(ILogger logger, ServiceOptions options) - { - _logger = logger; - _options = options; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - Directory.CreateDirectory(_options.LogDirectory); - _stdoutLogPath = Path.Combine(_options.LogDirectory, "backend-service.stdout.log"); - _stderrLogPath = Path.Combine(_options.LogDirectory, "backend-service.stderr.log"); - - LogStdOut("Service host starting backend process."); - LogStdOut($"Script: {_options.BackendScriptPath}"); - LogStdOut($"Working directory: {_options.WorkingDirectory}"); - - var powershellPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.Windows), - "System32", - "WindowsPowerShell", - "v1.0", - "powershell.exe"); - - var startInfo = new ProcessStartInfo - { - FileName = powershellPath, - Arguments = $"-NoProfile -ExecutionPolicy Bypass -File \"{_options.BackendScriptPath}\"", - WorkingDirectory = _options.WorkingDirectory, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - _backendProcess = new Process { StartInfo = startInfo }; - if (!_backendProcess.Start()) - { - throw new InvalidOperationException("Failed to start backend process."); - } - - LogStdOut($"Backend process started with PID {_backendProcess.Id}."); - _logger.LogInformation("Backend process started with PID {Pid}.", _backendProcess.Id); - - var stdoutPump = PumpStreamAsync(_backendProcess.StandardOutput, LogStdOut, stoppingToken); - var stderrPump = PumpStreamAsync(_backendProcess.StandardError, LogStdErr, stoppingToken); - - try - { - await _backendProcess.WaitForExitAsync(stoppingToken); - var exitCode = _backendProcess.ExitCode; - LogStdErr($"Backend process exited unexpectedly with code {exitCode}."); - _logger.LogError("Backend process exited unexpectedly with code {ExitCode}.", exitCode); - Environment.ExitCode = exitCode == 0 ? 1 : exitCode; - throw new InvalidOperationException( - $"Backend process ended unexpectedly. Service host exit code: {Environment.ExitCode}."); - } - catch (OperationCanceledException) - { - LogStdOut("Service stop requested."); - } - finally - { - await Task.WhenAll(stdoutPump, stderrPump); - } - } - - public override async Task StopAsync(CancellationToken cancellationToken) - { - if (_backendProcess is { HasExited: false }) - { - try - { - LogStdOut("Stopping backend process."); - _backendProcess.Kill(entireProcessTree: true); - } - catch (Exception ex) - { - LogStdErr($"Failed to stop backend process cleanly: {ex.Message}"); - _logger.LogError(ex, "Failed to stop backend process cleanly."); - } - } - - await base.StopAsync(cancellationToken); - } - - private async Task PumpStreamAsync( - StreamReader reader, - Action sink, - CancellationToken stoppingToken) - { - while (!stoppingToken.IsCancellationRequested) - { - var line = await reader.ReadLineAsync(); - if (line is null) - { - break; - } - - sink(line); - } - } - - private void LogStdOut(string message) - { - WriteLog(_stdoutLogPath, message); - } - - private void LogStdErr(string message) - { - WriteLog(_stderrLogPath, message); - } - - private void WriteLog(string path, string message) - { - var stamp = DateTimeOffset.Now.ToString("yyyy-MM-dd HH:mm:ss"); - var line = $"[{stamp}] {message}{Environment.NewLine}"; - lock (_logLock) - { - File.AppendAllText(path, line); - } - } -} diff --git a/service_host/ScreenJob.WindowsServiceHost/Program.cs b/service_host/ScreenJob.WindowsServiceHost/Program.cs deleted file mode 100644 index 66177f6..0000000 --- a/service_host/ScreenJob.WindowsServiceHost/Program.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using ScreenJob.WindowsServiceHost; - -var options = ServiceOptions.Parse(args); - -Host.CreateDefaultBuilder(args) - .UseWindowsService(serviceOptions => - { - serviceOptions.ServiceName = "ScreenJobBackend"; - }) - .ConfigureServices(services => - { - services.AddSingleton(options); - services.AddHostedService(); - }) - .Build() - .Run(); diff --git a/service_host/ScreenJob.WindowsServiceHost/ScreenJob.WindowsServiceHost.csproj b/service_host/ScreenJob.WindowsServiceHost/ScreenJob.WindowsServiceHost.csproj deleted file mode 100644 index f05a01a..0000000 --- a/service_host/ScreenJob.WindowsServiceHost/ScreenJob.WindowsServiceHost.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - net10.0-windows - enable - enable - Exe - - - - - - diff --git a/service_host/ScreenJob.WindowsServiceHost/ServiceOptions.cs b/service_host/ScreenJob.WindowsServiceHost/ServiceOptions.cs deleted file mode 100644 index 9c940b9..0000000 --- a/service_host/ScreenJob.WindowsServiceHost/ServiceOptions.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace ScreenJob.WindowsServiceHost; - -internal sealed record ServiceOptions( - string BackendScriptPath, - string WorkingDirectory, - string LogDirectory) -{ - public static ServiceOptions Parse(string[] args) - { - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - - for (var i = 0; i < args.Length; i++) - { - var raw = args[i]; - if (!raw.StartsWith("--", StringComparison.Ordinal)) - { - continue; - } - - var key = raw[2..]; - if (string.IsNullOrWhiteSpace(key)) - { - continue; - } - - if (i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal)) - { - map[key] = args[++i]; - } - else - { - map[key] = "true"; - } - } - - if (!map.TryGetValue("backend-script", out var backendScript) || string.IsNullOrWhiteSpace(backendScript)) - { - throw new ArgumentException("Missing required argument: --backend-script ."); - } - - if (!Path.IsPathRooted(backendScript)) - { - throw new ArgumentException("The --backend-script value must be an absolute path."); - } - - if (!File.Exists(backendScript)) - { - throw new FileNotFoundException("Backend script not found.", backendScript); - } - - if (!map.TryGetValue("working-dir", out var workingDir) || string.IsNullOrWhiteSpace(workingDir)) - { - workingDir = Path.GetDirectoryName(backendScript) - ?? throw new ArgumentException("Could not resolve working directory from backend script path."); - } - - if (!Path.IsPathRooted(workingDir)) - { - throw new ArgumentException("The --working-dir value must be an absolute path."); - } - - if (!map.TryGetValue("log-dir", out var logDir) || string.IsNullOrWhiteSpace(logDir)) - { - logDir = Path.Combine(workingDir, "screenjob_runs", "service"); - } - - if (!Path.IsPathRooted(logDir)) - { - throw new ArgumentException("The --log-dir value must be an absolute path."); - } - - return new ServiceOptions( - Path.GetFullPath(backendScript), - Path.GetFullPath(workingDir), - Path.GetFullPath(logDir)); - } -} diff --git a/src/agent.py b/src/agent.py index 3baac92..8aa5f99 100644 --- a/src/agent.py +++ b/src/agent.py @@ -6,12 +6,14 @@ import json import logging import os import re +import signal import sys import subprocess import threading import time import traceback from datetime import datetime +from pathlib import Path from typing import Any, Callable from openai import OpenAI @@ -30,55 +32,47 @@ else: _PYAUTOGUI_IMPORT_ERROR = None +class RunCancelled(Exception): + pass + + CORE_OPERATING_DOCTRINE = """ Core operating doctrine: -1) Use tools to act. Do not claim actions without tool calls. -2) Maintain a live mental model of the current surface kind, foreground app identity, likely focus, pending dialog state, browser workflow state when relevant, pointer state when relevant, and the observed outcome of the last action. -3) Work in this loop: classify -> choose control channel -> execute one meaningful transition -> verify. -4) First classify, then act. Classify the current UI into window/app/dialog/browser/system surface before choosing the next tool family. -5) Tool precedence: native window/dialog/element tool when confidence is high; execute_command for deterministic launches/checks; see_screen and enhance for grounding; raw mouse/keyboard fallback only when native routes are unavailable or weaker. -6) Verify before risky action. Re-check after state-changing or ambiguous action. If uncertain, observe again. -7) Keep tool arguments valid JSON and concise. +1) Use tools to act. Do not claim actions you did not take. +2) Work in this loop: observe -> choose the best tool -> make one meaningful move -> verify. +3) Prefer deterministic and native routes when they fit; use pixel input as fallback. +4) Verify risky or ambiguous changes before repeating them. +5) Keep tool arguments valid JSON and concise. """ WINDOWS_ENVIRONMENT_RULES = """ Windows environment rules: -1) Treat command-launched apps or URLs as background until get_active_window, wait_for_focus_change, wait_for_window, or see_screen proves focus changed. -2) Use get_active_window, list_windows, find_window, focus_window, detect_dialog, and wait helpers to reason about focus transfer, modal ownership, background launches, Start menu/taskbar surfaces, Explorer pickers, and hidden windows. -3) Recognize common Windows surfaces explicitly: normal top-level app window, #32770 modal dialog, Explorer open/save picker, taskbar/start menu, context menu, toast notification, browser window chrome, browser page content, permission prompt, and secure desktop/UAC interruption. -4) If focus may be wrong, verify with get_active_window or a native wait/focus helper before typing or clicking. -5) If pointer state matters, call get_cursor_position before move_mouse or drag. Use move_mouse and drag only after confirming the intended region. -6) Prefer non-visual verification when available: clipboard_get for copy/cut, get_active_window or window tools for focus/open/close changes, element tools for enabled/visible state, and command stdout for deterministic shell actions. +1) Do not assume command-launched apps or URLs became foreground; verify focus first. +2) Use window, dialog, and wait tools to handle focus changes, modal dialogs, and picker flows. +3) If focus may be wrong, verify before typing or clicking. +4) Prefer non-visual verification when available. """ BROWSER_WORKFLOW_RULES = """ Browser workflow rules: -1) Distinguish browser chrome from page content. Address bar, tab strip, downloads controls, permission prompts, and file dialogs are browser or system surfaces, not page content. -2) For browser-heavy tasks, prefer native window/dialog/element operations for focus, tab/window selection, file upload/download dialogs, and browser-owned confirmation surfaces when exposed. -3) Use screenshots for page-content grounding and for custom-drawn browser surfaces that native tooling cannot read. -4) For upload/download flows, check first for dialog or browser-owned downloads UI before repeating pixel clicks. +1) Distinguish browser chrome from page content. +2) Use native helpers for browser dialogs and focus when available. +3) Use screenshots for page-content grounding. """ DIALOG_HANDLING_RULES = """ Dialog-handling rules: -1) If an unexpected modal or confirmation dialog appears, pause the prior plan and resolve or dismiss the modal first. -2) When a save/open/confirm dialog is likely, prefer detect_dialog, dialog_set_filename, dialog_action, wait_for_window, wait_for_dialog_close, list_ui_elements, and set_ui_element_value before coordinate clicking. -3) Common dialog actions include Open, Save, Cancel, OK, Yes, and No. Do not assume the dialog is gone until verified. -4) If secure desktop or UAC is detected or strongly suspected, report a blocked state explicitly instead of blind retries. -5) If a control is disabled, hidden, or not exposed natively, fall back to screenshot reasoning and raw input only after recording that fallback. +1) Resolve unexpected modals before resuming the old plan. +2) Prefer dialog tools over coordinate clicks for open/save/confirm flows. +3) If secure desktop or UAC appears, report the block instead of blind retries. """ COMPLETION_VERIFICATION_RULES = """ Completion and verification rules: -1) Use see_screen at a balanced cadence: before a new UI phase, after navigation or layout changes, after actions that could fail silently, and before task_complete. Do not spam screenshots after every deterministic micro-step. -2) Treat enhance as the default follow-up when a target is small, dense, visually similar, or text-heavy. For tiny controls use enhance(coordinate, region="small", mode="ui"). For tiny text use mode="text". -3) Never infer success from intent. Never spam repeated clicks on the same coordinate; switch strategy and re-observe. -4) Do not invent new subgoals. Unless the objective explicitly asks for it, do not search for saved files, browse the filesystem, reopen apps, or otherwise expand the task boundary after the requested result is already achieved. -5) After copy, paste, save, close, upload, download, or navigation milestones, prefer verify-and-finish over extra exploration. -6) When objective is fully complete, call task_complete(return="...", data=...). -7) Before task_complete, do a fresh verification pass with see_screen and add enhance when the proof is small, dense, or text-heavy. -8) In task_complete data, include explicit verification text such as data.observed_result describing what you just observed. -9) If runtime evidence says the objective is likely already satisfied, do one fresh verification pass and then call task_complete unless you observe a concrete contradiction. +1) Re-observe with enhance when you need fresh grounding, zoom, or text clarity. +2) Do not invent extra subgoals after the requested result is achieved. +3) When the objective is complete, call task_complete(return="...", data=...). +4) Include clear verification text in data.observed_result. """ @@ -120,17 +114,18 @@ def build_initial_action_prompt( "Use classify -> choose control channel -> execute one meaningful transition -> verify.\n" "Prefer native window/dialog/element tools when they fit the current surface; use screenshots and raw pointer input as fallback.\n" "Prefer execute_command for deterministic actions.\n" - "Treat command-launched apps or URLs as background until get_active_window, wait_for_focus_change, wait_for_window, or see_screen proves they took focus.\n" + "Treat command-launched apps or URLs as background until the injected foreground-window context, wait_for_focus_change, wait_for_window, or the latest retained visual proves they took focus.\n" "For modifier shortcuts, use a single press_key combo (example: win+r).\n" f"{_prohibited_key_combo_prompt(prohibited_key_combos)}" "Explicitly watch for #32770 dialogs, Explorer open/save pickers, browser download/upload flows, taskbar/start menu focus traps, context menus, disabled controls, and permission prompts.\n" - "If focus or the foreground app may be wrong, call get_active_window, find_window, focus_window, or a focus wait helper.\n" + "If focus or the foreground app may be wrong, rely on the injected foreground-window context or call find_window, focus_window, or a focus wait helper.\n" "If an unexpected modal appears, pause the prior plan and resolve the modal first.\n" "Do not invent new subgoals. Prefer non-visual verification when available.\n" "Use wait_for_focus_change when focus transfer is expected but not yet confirmed.\n" "When a fresh focus check or a clear retained visual already proves the target editor or field is ready, act directly; do not re-capture the screen just to reconfirm an obvious large input area.\n" - "You may use more than one tool in one step when that improves certainty, such as get_active_window plus detect_dialog, see_screen then enhance, or click then see_screen.\n" - "When done, do a fresh verification pass with see_screen and add enhance if the proof is small or text-heavy.\n" + "You may use more than one tool in one step when that improves certainty, such as window context plus detect_dialog, enhance after a fresh screen change, or click then use the retained current screen.\n" + "After important tool results, verify the expected UI or focus change before repeating the same action or chaining another risky action.\n" + "When done, rely on the latest retained screen and add enhance if the proof is small or text-heavy.\n" "Then call task_complete(return=..., data={\"observed_result\": ...}).\n" "Include useful structured output in data.", ) @@ -139,13 +134,13 @@ def build_initial_action_prompt( def build_no_tool_prompt(prohibited_key_combos: list[str] | tuple[str, ...] | set[str] | None = None) -> str: return _compose_prompt( "No function call was returned. Recover by re-observing the current desktop state instead of guessing.\n" - "Start by classifying the surface. Use get_active_window, detect_dialog, find_window, see_screen, enhance, get_cursor_position, or clipboard_get according to what is missing.\n" + "Start by classifying the surface. Use the injected foreground-window context, detect_dialog, find_window, enhance, get_cursor_position, or clipboard_get according to what is missing.\n" "Rebuild the live mental model, identify what changed, and choose the next safe action only after confirming the target and likely result.\n" "Route toward native window/dialog/element tools before repeating raw clicks on Windows surfaces.\n" f"{_prohibited_key_combo_prompt(prohibited_key_combos)}" "Do not assume execute_command launches changed the foreground window; verify focus before typing.\n" "If a modal, picker, or browser download/upload surface is likely, resolve that first.\n" - "Before task_complete, do a fresh verification pass with see_screen, add enhance if needed, and include data.observed_result.", + "Before task_complete, use the latest retained screen, add enhance if needed, and include data.observed_result.", ) @@ -174,15 +169,15 @@ def build_context_compaction_prompt( "Retained visual observations:\n" f"{visual_lines}\n" "Treat prior reasoning as stale, but do not throw away the retained visuals below.\n" - "The retained visuals already represent the latest image tool calls and their results. Reuse them first; do not call see_screen again only because compaction happened.\n" + "The retained visuals already represent the latest image observations and current screen snapshots. Reuse them first; do not ask for another visual just because compaction happened.\n" "Determine the current surface kind, foreground app, likely focus, pending dialog/browser state, and what changed since the last action before acting.\n" "Use classify -> choose control channel -> execute one meaningful transition -> verify.\n" "Follow observe -> decide -> act -> verify.\n" f"{_prohibited_key_combo_prompt(prohibited_key_combos)}" - "Re-observe with see_screen only when the retained visuals are stale or the UI likely changed. Add enhance for small or text-heavy details, and use get_active_window, detect_dialog, wait helpers, clipboard_get, or command stdout when they are the better verification channel.\n" + "Re-observe with enhance only when the retained visuals are stale or you need zoom/text clarity. Use the injected foreground-window context, detect_dialog, wait helpers, clipboard_get, or command stdout when they are the better verification channel.\n" "If a fresh focus check or retained visual already proves a text field or editor is ready, act without demanding another screenshot.\n" "Treat execute_command app or URL launches as background until focus is explicitly verified.\n" - "Use tools only. Finish only after a fresh verification pass with see_screen and explicit data.observed_result in task_complete.", + "Use tools only. Finish only after using the latest retained screen and explicit data.observed_result in task_complete.", ) @@ -196,10 +191,10 @@ def build_blocked_action_prompt( f"The last action using {tool_name} was blocked or unreliable. Do not retry blindly.\n" "Re-anchor on the live desktop state first: classify the current surface, then choose the best tool family.\n" "If this looks like a dialog, picker, permission prompt, or browser-owned confirmation surface, route to detect_dialog, dialog_action, dialog_set_filename, wait_for_dialog_close, list_ui_elements, or focus/window tools before pixel retries.\n" - "If focus or the foreground app may be wrong, call get_active_window, find_window, or focus_window.\n" + "If focus or the foreground app may be wrong, rely on the injected foreground-window context, find_window, or focus_window.\n" "If pointer placement matters, call get_cursor_position before move_mouse or drag.\n" "If copy success matters, call clipboard_get instead of assuming it worked.\n" - "If execute_command launched an app or URL, do not assume it is foreground until get_active_window, wait_for_focus_change, wait_for_window, or see_screen confirms it.\n" + "If execute_command launched an app or URL, do not assume it is foreground until the injected foreground-window context, wait_for_focus_change, wait_for_window, or the latest retained visual confirms it.\n" f"{_prohibited_key_combo_prompt(prohibited_key_combos)}" "If secure desktop or UAC is suspected, stop blind retries and report the blocked state explicitly.\n" "Switch strategy after the fresh classification: native control instead of pixels, keyboard instead of mouse, mouse instead of keyboard, commands instead of UI, UI instead of commands, or finish if the job is already done.\n" @@ -212,11 +207,11 @@ def build_observation_loop_prompt( repeated_steps: int = 3, prohibited_key_combos: list[str] | tuple[str, ...] | set[str] | None = None, ) -> str: - summary_text = f" Current foreground window: {window_summary}." if window_summary else "" + summary_text = f" Current injected foreground-window context: {window_summary}." if window_summary else "" return _compose_prompt( f"You have already re-observed the same stable window for {repeated_steps} step(s) without making progress." f"{summary_text}\n" - "Do not keep calling broad observation tools like see_screen or get_active_window on the same unchanged state.\n" + "Do not keep calling broad observation tools like enhance on the same unchanged state.\n" "Change method now: use a native window/dialog/element tool for this surface, interact with the visible control, resolve the modal, or finish if you already have proof.\n" "If the requested navigation or status check already happened, do one final verification pass and then call task_complete instead of waiting or re-clicking.\n" f"{_prohibited_key_combo_prompt(prohibited_key_combos)}" @@ -244,7 +239,7 @@ def build_finish_likely_prompt( return _compose_prompt( "Runtime completion evidence indicates the objective is likely already satisfied.\n" f"Evidence: {evidence_summary}\n" - "Do one fresh verification pass now: call see_screen, add enhance only if the proof is small or text-heavy, then call task_complete.\n" + "Do one fresh verification pass now using the latest retained screen; add enhance only if the proof is small or text-heavy, then call task_complete.\n" f"{_prohibited_key_combo_prompt(prohibited_key_combos)}" "Do not reopen menus, repeat save/export/download actions, or re-search the filesystem unless a new contradiction appears.", ) @@ -254,7 +249,6 @@ ALL_TOOL_NAMES: tuple[str, ...] = ( "task_complete", "execute_command", "sleep", - "see_screen", "enhance", "list_windows", "find_window", @@ -280,10 +274,9 @@ ALL_TOOL_NAMES: tuple[str, ...] = ( "clipboard_get", "clipboard_set", "get_cursor_position", - "get_active_window", ) PROTECTED_TOOL_NAMES = {"task_complete"} -VISUAL_TOOL_NAMES = {"see_screen", "enhance"} +VISUAL_TOOL_NAMES = {"enhance"} WINDOW_TOOL_NAMES = { "list_windows", "find_window", @@ -291,7 +284,6 @@ WINDOW_TOOL_NAMES = { "close_window", "wait_for_window", "wait_for_focus_change", - "get_active_window", } DIALOG_TOOL_NAMES = { "detect_dialog", @@ -315,7 +307,6 @@ OBSERVATION_NON_PROGRESS_TOOL_NAMES = VISUAL_TOOL_NAMES | { "find_window", "wait_for_window", "wait_for_focus_change", - "get_active_window", "detect_dialog", "wait_for_dialog_close", "list_ui_elements", @@ -327,8 +318,8 @@ OBSERVATION_NON_PROGRESS_TOOL_NAMES = VISUAL_TOOL_NAMES | { WINDOWS_ONLY_TOOL_NAMES = WINDOW_TOOL_NAMES | DIALOG_TOOL_NAMES | UI_ELEMENT_TOOL_NAMES MAX_ACTION_SIGNATURE_ATTEMPTS = 3 MAX_STABLE_OBSERVATION_STEPS = 3 -FINISH_LIKELY_OBSERVATION_TOOLS = {"see_screen", "enhance", "get_active_window", "detect_dialog"} -BROAD_REOBSERVATION_TOOL_NAMES = {"see_screen", "get_active_window", "detect_dialog", "list_windows", "sleep"} +FINISH_LIKELY_OBSERVATION_TOOLS = {"enhance", "detect_dialog", "find_window", "focus_window", "wait_for_focus_change"} +BROAD_REOBSERVATION_TOOL_NAMES = {"enhance", "detect_dialog", "list_windows", "sleep"} def normalize_disabled_tools(tool_names: set[str] | list[str] | tuple[str, ...] | None) -> list[str]: @@ -382,6 +373,7 @@ class ScreenJobAgent: self.click_history: list[tuple[int, int, float]] = [] self.disabled_tools = set(normalize_disabled_tools(options.disable_tools)) self.recent_tool_summaries: list[str] = [] + self.last_tool_call_name: str | None = None self.last_context_compact_step = 0 self.visual_context_messages: list[dict[str, Any]] = [] self.visual_context_overflow_pending = False @@ -469,7 +461,7 @@ class ScreenJobAgent: "name": "task_complete", "description": ( "Call this only when the job objective is fully done and freshly verified. " - "Before finishing, call see_screen and add enhance if the proof is small or text-heavy. " + "Before finishing, use the latest retained screen and add enhance if the proof is small or text-heavy. " "Include explicit verification text such as data.observed_result." ), "parameters": { @@ -489,7 +481,7 @@ class ScreenJobAgent: "description": ( "Run a shell command and return stdout/stderr/exit code. " "Prefer this for deterministic operations like opening URLs. " - "Do not assume a launched app or URL took foreground focus until get_active_window or see_screen confirms it. " + "Do not assume a launched app or URL took foreground focus until the injected foreground-window context or the latest retained visual confirms it. " "Do not use recursive file-search or reveal commands unless the objective explicitly asks to locate an output." ), "parameters": { @@ -517,24 +509,12 @@ class ScreenJobAgent: "additionalProperties": False, }, }, - { - "type": "function", - "name": "see_screen", - "description": ( - "Capture full screen with coordinate grid overlay. " - "Use before a new UI phase, after state changes, when uncertain, and before task_complete." - ), - "parameters": { - "type": "object", - "properties": {}, - "additionalProperties": False, - }, - }, { "type": "function", "name": "enhance", "description": ( - "Create enhanced zoom around a coordinate for readability and precise targeting. " + "Capture the current screen. With no coordinate, return a full-screen view with a coordinate grid. " + "With a coordinate, create an enhanced zoom for readability and precise targeting. " "Prefer this for tiny, dense, visually similar, or text-heavy targets." ), "parameters": { @@ -551,7 +531,7 @@ class ScreenJobAgent: }, "region": { "type": "string", - "enum": ["small", "medium", "large"], + "enum": ["full", "small", "medium", "large"], }, "mode": { "type": "string", @@ -559,10 +539,9 @@ class ScreenJobAgent: }, "scale": { "type": ["integer", "string"], - "description": "Zoom factor from 2 to 6. Defaults by region.", + "description": "Zoom factor from 2 to 6. Defaults by region when using a coordinate.", }, }, - "required": ["coordinate"], "additionalProperties": False, }, }, @@ -911,7 +890,7 @@ class ScreenJobAgent: "name": "drag", "description": ( "Drag the mouse from one absolute screen coordinate to another. " - "Confirm the intended control region with see_screen or enhance before dragging." + "Confirm the intended control region with enhance or the latest retained screen before dragging." ), "parameters": { "type": "object", @@ -1003,23 +982,8 @@ class ScreenJobAgent: "additionalProperties": False, }, }, - { - "type": "function", - "name": "get_active_window", - "description": ( - "Return metadata for the current foreground window to verify focus and active app, surface kind, " - "browser state, and dialog classification." - ), - "parameters": { - "type": "object", - "properties": {}, - "additionalProperties": False, - }, - }, ] optional_native_tools = set(WINDOWS_ONLY_TOOL_NAMES) - if self._is_windows_host() and not self._native_control_tools_enabled(): - optional_native_tools = optional_native_tools - {"get_active_window"} if not self._is_windows_host() or not self._native_control_tools_enabled(): return [ tool @@ -1086,7 +1050,7 @@ class ScreenJobAgent: def _native_automation_mode(self) -> str: mode = str(self.options.native_automation_mode or "prefer").strip().lower() - if mode not in {"off", "prefer", "require_fallback"}: + if mode not in {"off", "prefer"}: return "prefer" return mode @@ -1177,12 +1141,12 @@ class ScreenJobAgent: if surface_kind == "modal_dialog": return ["detect_dialog", "dialog_action", "list_ui_elements"] if surface_kind == "browser_window": - return ["get_active_window", "detect_dialog", "see_screen", "enhance"] + return ["detect_dialog", "enhance"] if surface_kind == "system_surface": - return ["get_active_window", "see_screen", "enhance"] + return ["enhance"] if dialog_kind != "none": return ["detect_dialog", "dialog_action"] - return ["get_active_window", "see_screen", "enhance"] + return ["enhance"] def _build_target_handle( self, @@ -1327,7 +1291,7 @@ class ScreenJobAgent: "blocking_reason": blocking_reason, "surface_kind": str(self.last_surface_state.get("surface_kind") or "unknown"), "dialog_kind": str(self.last_surface_state.get("dialog_kind") or "none"), - "recommended_next_tools": ["get_active_window", "see_screen", "enhance"], + "recommended_next_tools": ["enhance"], "native_automation_mode": self._native_automation_mode(), } @@ -1357,19 +1321,27 @@ class ScreenJobAgent: path = str(meta.get("path") or "").strip() captured_at = str(meta.get("captured_at") or "").strip() if tool_name == "enhance": - source = meta.get("source_coord") if isinstance(meta.get("source_coord"), dict) else {} - source_x = self._parse_int(source.get("x"), default=0) - source_y = self._parse_int(source.get("y"), default=0) region = str(meta.get("region") or "small").strip() mode = str(meta.get("mode") or "ui").strip() scale = self._parse_int(meta.get("scale"), default=0) - pieces = [f"enhance ok at=({source_x},{source_y})", f"region={region}", f"mode={mode}"] - if scale > 0: - pieces.append(f"scale={scale}") + if bool(meta.get("full_screen")): + size = meta.get("screen_size") if isinstance(meta.get("screen_size"), dict) else {} + width = self._parse_int(size.get("width"), default=0) + height = self._parse_int(size.get("height"), default=0) + pieces = ["enhance ok full_screen", f"region={region}", f"mode={mode}"] + if width > 0 and height > 0: + pieces.append(f"size={width}x{height}") + else: + source = meta.get("source_coord") if isinstance(meta.get("source_coord"), dict) else {} + source_x = self._parse_int(source.get("x"), default=0) + source_y = self._parse_int(source.get("y"), default=0) + pieces = [f"enhance ok at=({source_x},{source_y})", f"region={region}", f"mode={mode}"] + if scale > 0: + pieces.append(f"scale={scale}") else: width = self._parse_int(meta.get("width"), default=0) height = self._parse_int(meta.get("height"), default=0) - pieces = ["see_screen ok"] + pieces = ["visual ok"] if width > 0 and height > 0: pieces.append(f"size={width}x{height}") if captured_at: @@ -1405,7 +1377,9 @@ class ScreenJobAgent: self.visual_context_overflow_pending = bool(self.visual_context_messages) self.visual_context_messages = [] return - if len(self.visual_context_messages) > budget: + # Keep a small amount of headroom so a single new snapshot does not + # force a prompt rebuild on every step once the visual budget is full. + if len(self.visual_context_messages) > budget + 1: self.visual_context_overflow_pending = True self.visual_context_messages = self._latest_visual_context_entries( self.visual_context_messages, @@ -1633,15 +1607,15 @@ class ScreenJobAgent: lowered_stdout = stdout.lower() if any(token in lowered_stdout for token in ("missing", "not found", "cannot find", "false")): self._clear_finish_likely(f'Command verification contradicted completion for "{target_filename}".') - elif tool_name == "see_screen": + elif tool_name == "enhance": meta = result.get("meta") if isinstance(result.get("meta"), dict) else {} visual_signature = str(meta.get("visual_signature") or "").strip() post_signature = str(self.finish_likely_state.get("post_completion_visual_signature") or "").strip() - if bool(self.finish_likely_state.get("active")) and visual_signature and post_signature and visual_signature == post_signature: + if bool(meta.get("full_screen")) and bool(self.finish_likely_state.get("active")) and visual_signature and post_signature and visual_signature == post_signature: recorded = self._record_completion_evidence( kind="verification_screenshot_matches_post_completion_state", category="independent_verifier", - summary="Fresh verification screenshot matches the stable post-completion state.", + summary="Fresh full-screen verification matches the stable post-completion state.", detail={"visual_signature": visual_signature}, ) if recorded is not None: @@ -1681,23 +1655,17 @@ class ScreenJobAgent: click_count = clamp(self._parse_int(args.get("click_count"), default=1), 1, 5) return { "signature": f"click:{x}:{y}:{button}:{click_count}", - "required_verifiers": {"see_screen", "enhance"}, - "hint": "Verify the visible UI change with see_screen or enhance before repeating the same click.", + "required_verifiers": {"enhance"}, + "hint": "Verify the visible UI change with enhance before repeating the same click.", } if tool_name == "type": - text = str(args.get("text", "")) - preview = text[:60].replace("\n", "\\n") - return { - "signature": f"type:{preview}", - "required_verifiers": {"see_screen", "enhance", "get_active_window"}, - "hint": "Verify where the text landed before typing the same content again.", - } + return None if tool_name == "scroll": amount = self._parse_int(args.get("amount"), default=0) direction = str(args.get("direction", "") or "").strip().lower() return { "signature": f"scroll:{amount}:{direction}", - "required_verifiers": {"see_screen", "enhance"}, + "required_verifiers": {"enhance"}, "hint": "Verify the page or panel moved before repeating the same scroll.", } if tool_name == "drag": @@ -1710,7 +1678,7 @@ class ScreenJobAgent: button = str(args.get("button", "left") or "left").strip().lower() or "left" return { "signature": f"drag:{start_x}:{start_y}:{end_x}:{end_y}:{button}", - "required_verifiers": {"see_screen", "enhance", "get_cursor_position"}, + "required_verifiers": {"enhance", "get_cursor_position"}, "hint": "Verify the dragged UI state before repeating the same drag path.", } if tool_name == "press_key": @@ -1725,23 +1693,11 @@ class ScreenJobAgent: "required_verifiers": {"clipboard_get"}, "hint": "Verify the clipboard with clipboard_get before retrying the same copy or cut shortcut.", } - if combo_text in {"ctrl+v", "ctrl+s", "ctrl+w", "alt+f4"}: - return { - "signature": f"press_key:{combo_text}", - "required_verifiers": {"see_screen", "enhance", "get_active_window"}, - "hint": "Verify the save, paste, close, or window outcome before sending the same shortcut again.", - } + if combo_text in {"ctrl+v", "ctrl+s", "ctrl+w", "alt+f4", "enter"}: + return None if {"alt", "tab"} <= combo_set or {"win", "tab"} <= combo_set or {"win", "r"} <= combo_set: - return { - "signature": f"press_key:{combo_text}", - "required_verifiers": {"get_active_window"}, - "hint": "Verify the foreground window with get_active_window before repeating the same focus or open shortcut.", - } - return { - "signature": f"press_key:{combo_text}", - "required_verifiers": {"see_screen", "enhance"}, - "hint": "Verify the visible UI change before repeating the same shortcut.", - } + return None + return None if tool_name == "execute_command": command = str(args.get("command", "") or "").strip() if not self._is_background_launch_command(command): @@ -1749,10 +1705,10 @@ class ScreenJobAgent: normalized = re.sub(r"\s+", " ", command).strip().lower() return { "signature": f"execute_command:{normalized}", - "required_verifiers": {"get_active_window", "see_screen"}, + "required_verifiers": {"enhance"}, "hint": ( "Command-launched apps or URLs may stay in the background. " - "Verify focus with get_active_window or see_screen before retrying or typing." + "Verify focus with the injected foreground-window context or the latest retained screen before retrying or typing." ), } return None @@ -1992,23 +1948,6 @@ class ScreenJobAgent: state["required_verifiers"] = set(policy.get("required_verifiers") or []) state["surface_signature"] = self._current_surface_signature() - def _decorate_verification_result( - self, - result: dict[str, Any], - policy: dict[str, Any] | None, - ) -> dict[str, Any]: - if policy is None or not bool(result.get("ok")): - return result - verifiers = sorted(set(policy.get("required_verifiers") or [])) - if not verifiers: - return result - result["verification_required"] = True - result["verification_channels"] = verifiers - hint = str(policy.get("hint") or "").strip() - if hint: - result["next_step_hint"] = hint - return result - def _parse_px(self, value: Any) -> int: if value is None: return 0 @@ -2563,33 +2502,79 @@ class ScreenJobAgent: except Exception: return {"available": False} - def _tool_see_screen(self, _: dict[str, Any]) -> dict[str, Any]: - image, meta = self._capture_screen(with_grid=True) - out_path = self.artifacts.shots_dir / f"screen_step_{self.step:03d}.png" + def _store_visual_capture( + self, + image: Image.Image, + meta: dict[str, Any], + out_path: Any, + *, + message: str, + ) -> dict[str, Any]: + out_path = Path(out_path) self._save_image(image, out_path) data_url = image_to_data_url(image, "PNG") - visual_signature = self._compute_visual_signature(image) + resolved_meta = dict(meta) | {"path": str(out_path.resolve()), "visual_signature": visual_signature} self.last_visual_signature = visual_signature self.last_screen_data_url = data_url - self.last_screen_meta = meta | {"path": str(out_path.resolve()), "visual_signature": visual_signature} + self.last_screen_meta = resolved_meta return { "ok": True, "path": str(out_path.resolve()), - "meta": self.last_screen_meta, - "message": "Screen captured with coordinate grid.", + "meta": resolved_meta, + "message": message, } + def _capture_passive_visual_snapshot(self) -> dict[str, Any]: + image, meta = self._capture_screen(with_grid=True) + meta = dict(meta) | { + "captured_by": "auto", + "region": "full", + "mode": "ui", + "full_screen": True, + "grid": True, + "screen_size": {"width": image.width, "height": image.height}, + } + out_path = self.artifacts.shots_dir / f"auto_step_{self.step:03d}.png" + return self._store_visual_capture(image, meta, out_path, message="Automatic current screen snapshot captured.") + + def _tool_see_screen(self, _: dict[str, Any]) -> dict[str, Any]: + image, meta = self._capture_screen(with_grid=True) + meta = dict(meta) | { + "captured_by": "legacy_see_screen", + "region": "full", + "mode": "ui", + "full_screen": True, + "grid": True, + "screen_size": {"width": image.width, "height": image.height}, + } + out_path = self.artifacts.shots_dir / f"screen_step_{self.step:03d}.png" + return self._store_visual_capture(image, meta, out_path, message="Screen captured with coordinate grid.") + def _tool_enhance(self, args: dict[str, Any]) -> dict[str, Any]: - coord = args.get("coordinate") or {} + coord = args.get("coordinate") if isinstance(args.get("coordinate"), dict) else {} + has_coordinate = bool(coord) and "x" in coord and "y" in coord requested_x = self._parse_int(coord.get("x", 0), default=0) requested_y = self._parse_int(coord.get("y", 0), default=0) - region = str(args.get("region", "small") or "small").strip().lower() + region = str(args.get("region", "full" if not has_coordinate else "small") or ("full" if not has_coordinate else "small")).strip().lower() mode = str(args.get("mode", "ui") or "ui").strip().lower() - if region not in {"small", "medium", "large"}: - region = "small" + if region not in {"full", "small", "medium", "large"}: + region = "full" if not has_coordinate else "small" if mode not in {"ui", "text"}: mode = "ui" + if region == "full" or not has_coordinate: + image, meta = self._capture_screen(with_grid=True) + meta = dict(meta) | { + "captured_by": "enhance", + "requested_coord": {"x": requested_x, "y": requested_y} if has_coordinate else None, + "region": "full", + "mode": mode, + "full_screen": True, + "grid": True, + "screen_size": {"width": image.width, "height": image.height}, + } + out_path = self.artifacts.shots_dir / f"enhance_step_{self.step:03d}_full.png" + return self._store_visual_capture(image, meta, out_path, message="Full-screen view captured with coordinate grid.") region_half_by_preset = { "small": 96, @@ -2668,29 +2653,22 @@ class ScreenJobAgent: out_path = self.artifacts.enhance_dir / ( f"enhance_step_{self.step:03d}_{source_x}_{source_y}_{region}_{mode}_x{scale}.png" ) - self._save_image(enhanced, out_path) - data_url = image_to_data_url(enhanced, "PNG") - visual_signature = self._compute_visual_signature(enhanced) meta = { "captured_at": utc_now_iso(), + "captured_by": "enhance", "requested_coord": {"x": requested_x, "y": requested_y}, "source_coord": {"x": source_x, "y": source_y}, "source_box": {"left": left, "top": top, "right": right, "bottom": bottom}, "region": region, "mode": mode, "scale": scale, - "path": str(out_path.resolve()), "size": {"width": enhanced.width, "height": enhanced.height}, "target_pixel": {"x": cx, "y": cy}, "screen_size": {"width": width, "height": height}, "base_capture_meta": base_meta, - "visual_signature": visual_signature, } - self.last_visual_signature = visual_signature - self.last_screen_data_url = data_url - self.last_screen_meta = meta - return {"ok": True, "meta": meta, "message": "Enhanced view generated."} + return self._store_visual_capture(enhanced, meta, out_path, message="Enhanced view generated.") def _tool_click(self, args: dict[str, Any]) -> dict[str, Any]: coord = args.get("coordinate") or {} @@ -2734,7 +2712,7 @@ class ScreenJobAgent: "blocked": True, "error": ( "Repeated click loop detected at nearly same coordinate. " - "Do not retry blindly. Re-observe with see_screen/enhance, verify focus with get_active_window, " + "Do not retry blindly. Re-observe with enhance if needed, verify focus with the injected foreground-window context, " "and switch strategy before acting again." ), "clicked": {"x": x, "y": y}, @@ -3074,7 +3052,7 @@ class ScreenJobAgent: "timeout_seconds": timeout_seconds, "surface_kind": str(self.last_surface_state.get("surface_kind") or "unknown"), "dialog_kind": str(self.last_surface_state.get("dialog_kind") or "none"), - "recommended_next_tools": ["get_active_window", "see_screen", "enhance"], + "recommended_next_tools": ["enhance"], } target = self._build_target_handle( self._parse_int(window.get("hwnd"), default=0), @@ -3125,7 +3103,7 @@ class ScreenJobAgent: "timeout_seconds": timeout_seconds, "surface_kind": str(self.last_surface_state.get("surface_kind") or "unknown"), "dialog_kind": str(self.last_surface_state.get("dialog_kind") or "none"), - "recommended_next_tools": ["get_active_window", "see_screen"], + "recommended_next_tools": ["enhance"], } target = self._build_target_handle( self._parse_int(changed.get("hwnd"), default=0), @@ -3314,7 +3292,7 @@ class ScreenJobAgent: "error": "Timed out waiting for dialog to close.", "timeout_seconds": timeout_seconds, "dialog_hwnd": dialog_hwnd, - "recommended_next_tools": ["detect_dialog", "see_screen", "enhance"], + "recommended_next_tools": ["detect_dialog", "enhance"], } result = self._build_native_result( ok=True, @@ -3487,7 +3465,7 @@ class ScreenJobAgent: "ok": False, "error": "Timed out waiting for matching UI element.", "timeout_seconds": timeout_seconds, - "recommended_next_tools": ["list_ui_elements", "detect_dialog", "see_screen", "enhance"], + "recommended_next_tools": ["list_ui_elements", "detect_dialog", "enhance"], } self._emit( "ui_element_found", @@ -3647,6 +3625,28 @@ class ScreenJobAgent: ) return any(re.search(pattern, lowered) for pattern in launch_patterns) + def _terminate_process_tree(self, process: subprocess.Popen[str], *, force: bool) -> None: + try: + if os.name == "nt": + subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + check=False, + ) + return + pgid = os.getpgid(process.pid) + os.killpg(pgid, signal.SIGKILL if force else signal.SIGTERM) + except Exception: # noqa: BLE001 + try: + if force: + process.kill() + else: + process.terminate() + except Exception: # noqa: BLE001 + pass + def _tool_execute_command(self, args: dict[str, Any]) -> dict[str, Any]: command = str(args.get("command", "")).strip() if not command: @@ -3672,10 +3672,12 @@ class ScreenJobAgent: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + start_new_session=(os.name != "nt"), + creationflags=(getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0), ) while True: if self._is_cancelled(): - process.terminate() + self._terminate_process_tree(process, force=False) return { "ok": False, "cancelled": True, @@ -3685,7 +3687,7 @@ class ScreenJobAgent: if process.poll() is not None: break if (time.time() - started) > self.options.command_timeout: - process.kill() + self._terminate_process_tree(process, force=True) stdout, stderr = process.communicate(timeout=2) return { "ok": False, @@ -3712,16 +3714,13 @@ class ScreenJobAgent: result["focus_change_assumed"] = False result["next_step_hint"] = ( "Do not assume the launched app or URL is foreground. " - "Verify with get_active_window or see_screen before typing or retrying. " - "If get_active_window confirms the expected editor or dialog is focused, act directly instead of taking another screenshot first." + "Verify with the injected foreground-window context or the latest retained screen before typing or retrying. " + "If the injected foreground-window context confirms the expected editor or dialog is focused, act directly instead of demanding another visual first." ) return result except Exception as exc: # noqa: BLE001 if process is not None and process.poll() is None: - try: - process.kill() - except Exception: # noqa: BLE001 - pass + self._terminate_process_tree(process, force=True) return {"ok": False, "command": command, "error": f"{type(exc).__name__}: {exc}"} def _tool_task_complete(self, args: dict[str, Any]) -> dict[str, Any]: @@ -3737,9 +3736,25 @@ class ScreenJobAgent: self.final_data = data return {"ok": True, "return": return_text, "data": data} - def _dispatch_tool(self, name: str, args: dict[str, Any]) -> dict[str, Any]: + def _dispatch_tool( + self, + name: str, + args: dict[str, Any], + prior_tool_names: list[str] | None = None, + ) -> dict[str, Any]: if name in self.disabled_tools: return {"ok": False, "error": f"Tool '{name}' is disabled for this job."} + prior_tool_names = list(prior_tool_names or []) + previous_tool_name = self.last_tool_call_name + self.last_tool_call_name = name + if name == "sleep" and not prior_tool_names and previous_tool_name in {None, "sleep"}: + return { + "ok": False, + "blocked": True, + "blocked_reason": "sleep_requires_previous_tool_call", + "error": "Sleep cannot be the first tool call. Use another tool first, then pause if needed.", + "hint": "Use another observation or action tool before sleep, or rely on a tool's sleep_after_seconds option.", + } if not self._is_windows_host() and name in WINDOWS_ONLY_TOOL_NAMES: return {"ok": False, "error": f"Tool '{name}' is only available on Windows."} finish_likely_result = self._check_finish_likely_gate(name, args) @@ -3752,7 +3767,6 @@ class ScreenJobAgent: if gate_result is not None: return gate_result handlers = { - "see_screen": self._tool_see_screen, "enhance": self._tool_enhance, "list_windows": self._tool_list_windows, "find_window": self._tool_find_window, @@ -3776,7 +3790,6 @@ class ScreenJobAgent: "clipboard_get": self._tool_clipboard_get, "clipboard_set": self._tool_clipboard_set, "get_cursor_position": self._tool_get_cursor_position, - "get_active_window": self._tool_get_active_window, "type": self._tool_type, "press_key": self._tool_press_key, "sleep": self._tool_sleep, @@ -3791,7 +3804,6 @@ class ScreenJobAgent: self._mark_observation(name) if policy is not None and bool(result.get("ok")): self._note_action_attempt(policy) - result = self._decorate_verification_result(result, policy) return self._update_finish_likely_from_tool(name, args, result) def _safe_parse_args(self, raw: str | None) -> dict[str, Any]: @@ -3803,20 +3815,67 @@ class ScreenJobAgent: except Exception: # noqa: BLE001 return {"_raw": raw} + def _foreground_window_context_summary(self, window: dict[str, Any] | None) -> str: + if not isinstance(window, dict) or not bool(window.get("available")): + return "Foreground window unavailable." + title = str(window.get("title") or "").strip() or "(untitled)" + class_name = str(window.get("class_name") or "").strip() or "unknown" + executable_name = str(window.get("executable_name") or "").strip() or "unknown" + hwnd = self._parse_int(window.get("hwnd"), default=0) + surface_kind = self._surface_kind_from_window(window) + dialog_kind = self._dialog_kind_from_window(window) + browser_state = self._browser_workflow_state(window) or "none" + return ( + f'Foreground window title="{title}" | exe={executable_name} | class={class_name} | ' + f'hwnd={hwnd} | surface={surface_kind} | dialog={dialog_kind} | browser_state={browser_state}' + ) + + def _runtime_model_instructions(self) -> str: + window = self._get_active_window_info() + if isinstance(window, dict): + self.last_observed_window = dict(window) + self._update_surface_state(window=window, confidence=0.92, source_tool="runtime_window_context") + self.logger.info("⌂ %s", self._foreground_window_context_summary(window)) + return _compose_prompt( + SYSTEM_PROMPT, + "Runtime foreground-window context: " + self._foreground_window_context_summary(window), + "This foreground-window context is refreshed every step by the runtime. Trust it instead of calling a separate focus-inspection tool.", + "For routine text entry and submit flows, trust successful type and press_key results unless this window context or the screen meaningfully changes.", + ) + def _call_model(self, input_items: list[dict[str, Any]]) -> Any: effort = str(self.options.reasoning_effort or "medium").strip().lower() if effort not in {"low", "medium", "high"}: effort = "medium" - return self.client.responses.create( - model=self.options.model, - instructions=SYSTEM_PROMPT, - tools=self._tool_schemas(), - input=input_items, - previous_response_id=self.previous_response_id, - parallel_tool_calls=True, - max_tool_calls=8, - reasoning={"effort": effort}, - ) + + instructions = self._runtime_model_instructions() + response_box: dict[str, Any] = {} + error_box: dict[str, BaseException] = {} + + def _worker() -> None: + try: + response_box["response"] = self.client.responses.create( + model=self.options.model, + instructions=instructions, + tools=self._tool_schemas(), + input=input_items, + previous_response_id=self.previous_response_id, + parallel_tool_calls=True, + max_tool_calls=8, + reasoning={"effort": effort}, + ) + except BaseException as exc: # noqa: BLE001 + error_box["error"] = exc + + worker = threading.Thread(target=_worker, name=f"screenjob-openai-step-{self.step}", daemon=True) + worker.start() + while worker.is_alive(): + if self._is_cancelled(): + raise RunCancelled("Model call cancelled by user request.") + worker.join(timeout=0.05) + if "error" in error_box: + raise error_box["error"] + return response_box["response"] def _record_tool_summary(self, tool_name: str, result: dict[str, Any]) -> None: ok = bool(result.get("ok")) @@ -3863,11 +3922,6 @@ class ScreenJobAgent: y = position.get("y") if isinstance(x, int) and isinstance(y, int): summary = f"{summary} at=({x},{y})" - elif tool_name == "get_active_window": - window = result.get("window") if isinstance(result.get("window"), dict) else {} - title = str(window.get("title") or "").strip() - if title: - summary = f"{summary} title={title[:80]}" elif tool_name == "list_windows": summary = f"{summary} count={self._parse_int(result.get('count'), default=0)}" elif tool_name in {"find_window", "focus_window", "wait_for_window", "wait_for_focus_change"}: @@ -3886,7 +3940,7 @@ class ScreenJobAgent: exit_code = result.get("exit_code") if exit_code is not None: summary = f"{summary} exit_code={exit_code}" - elif tool_name in {"see_screen", "enhance"}: + elif tool_name == "enhance": meta = result.get("meta") if isinstance(result.get("meta"), dict) else {} path = str(meta.get("path") or result.get("path") or "").strip() if path: @@ -3920,11 +3974,13 @@ class ScreenJobAgent: return f"{tool_name} at=({x},{y})" if tool_name == "enhance": coord = args.get("coordinate") if isinstance(args.get("coordinate"), dict) else {} - x = self._parse_int((coord or {}).get("x"), default=0) - y = self._parse_int((coord or {}).get("y"), default=0) - region = str(args.get("region") or "small").strip().lower() or "small" + region = str(args.get("region") or ("small" if coord else "full")).strip().lower() or ("small" if coord else "full") mode = str(args.get("mode") or "ui").strip().lower() or "ui" - return f"{tool_name} at=({x},{y}) region={region} mode={mode}" + if coord: + x = self._parse_int((coord or {}).get("x"), default=0) + y = self._parse_int((coord or {}).get("y"), default=0) + return f"{tool_name} at=({x},{y}) region={region} mode={mode}" + return f"{tool_name} region={region} mode={mode}" if tool_name == "type": text = str(args.get("text") or "") preview = text.replace("\n", "\\n") @@ -3938,31 +3994,25 @@ class ScreenJobAgent: def _format_tool_result_log(self, tool_name: str, result: dict[str, Any]) -> str: status = "blocked" if bool(result.get("blocked")) else ("ok" if bool(result.get("ok")) else "error") parts = [f"{tool_name} {status}"] - if tool_name in {"see_screen", "enhance"}: + if tool_name == "enhance": meta = result.get("meta") if isinstance(result.get("meta"), dict) else {} path = str(meta.get("path") or result.get("path") or "").strip() - if tool_name == "see_screen": - width = self._parse_int(meta.get("width"), default=0) - height = self._parse_int(meta.get("height"), default=0) + if bool(meta.get("full_screen")): + size = meta.get("screen_size") if isinstance(meta.get("screen_size"), dict) else {} + width = self._parse_int(size.get("width"), default=0) + height = self._parse_int(size.get("height"), default=0) + parts.append("full_screen=true") if width > 0 and height > 0: parts.append(f"size={width}x{height}") - if tool_name == "enhance": + else: source = meta.get("source_coord") if isinstance(meta.get("source_coord"), dict) else {} parts.append( f"at=({self._parse_int(source.get('x'), default=0)},{self._parse_int(source.get('y'), default=0)})" ) - parts.append(f"region={str(meta.get('region') or 'small')}") - parts.append(f"mode={str(meta.get('mode') or 'ui')}") + parts.append(f"region={str(meta.get('region') or 'small')}") + parts.append(f"mode={str(meta.get('mode') or 'ui')}") if path: parts.append(f"path={path}") - elif tool_name == "get_active_window": - window = result.get("window") if isinstance(result.get("window"), dict) else {} - title = str(window.get("title") or "").strip() - class_name = str(window.get("class_name") or "").strip() - if title: - parts.append(f"title={title[:120]}") - if class_name: - parts.append(f"class={class_name}") elif tool_name in {"find_window", "focus_window", "wait_for_window", "wait_for_focus_change"}: window = result.get("window") if isinstance(result.get("window"), dict) else {} title = str(window.get("title") or "").strip() @@ -3999,9 +4049,6 @@ class ScreenJobAgent: parts.append(f"exit_code={exit_code}") if bool(result.get("background_launch_assumed")): parts.append("background_launch_assumed=true") - verification_channels = result.get("verification_channels") - if isinstance(verification_channels, list) and verification_channels: - parts.append("verify=" + ",".join(str(item) for item in verification_channels)) recommended_next_tools = result.get("recommended_next_tools") if isinstance(recommended_next_tools, list) and recommended_next_tools: parts.append("next=" + ",".join(str(item) for item in recommended_next_tools[:4])) @@ -4024,16 +4071,19 @@ class ScreenJobAgent: return " | ".join(parts) def _log_tool_call(self, tool_name: str, args: dict[str, Any]) -> None: - self.logger.info("Tool call: %s", self._format_tool_call_log(tool_name, args)) + self.logger.info("→ %s", self._format_tool_call_log(tool_name, args)) if self.options.pretty_logs: - self.logger.debug("Tool call detail (%s):\n%s", tool_name, json.dumps(args, ensure_ascii=False, indent=2)) + self.logger.debug(" args (%s)\n%s", tool_name, json.dumps(args, ensure_ascii=False, indent=2)) def _log_tool_result(self, tool_name: str, result: dict[str, Any]) -> None: - log_level = logging.INFO if bool(result.get("ok")) else logging.WARNING - self.logger.log(log_level, "Tool result: %s", self._format_tool_result_log(tool_name, result)) + ok = bool(result.get("ok")) + blocked = bool(result.get("blocked")) + prefix = "✓" if ok else ("⚠" if blocked else "✗") + log_level = logging.INFO if ok else logging.WARNING + self.logger.log(log_level, "%s %s", prefix, self._format_tool_result_log(tool_name, result)) if self.options.pretty_logs: self.logger.debug( - "Tool result detail (%s):\n%s", + " result (%s)\n%s", tool_name, json.dumps(result, ensure_ascii=False, indent=2), ) @@ -4107,10 +4157,14 @@ class ScreenJobAgent: str((entry.get("meta") or {}).get("path") or "") for entry in self._latest_visual_context_entries(self.visual_context_messages) ] + reason_label = { + "decay": "context decay", + "visual_budget": "visual budget overflow", + }.get(rebuild_reason, rebuild_reason.replace("_", " ")) self.logger.info( - "Compacted model context at step %d due to %s. retained_visuals=%d", + "Context compacted at step %d (reason=%s, retained_visuals=%d)", self.step, - rebuild_reason, + reason_label, len(retained_paths), ) if self.options.pretty_logs and retained_paths: @@ -4136,6 +4190,7 @@ class ScreenJobAgent: self.objective = job self.completion_evidence = {} self.last_observed_window = None + self.last_tool_call_name = None self.finish_likely_state.update( { "active": False, @@ -4169,7 +4224,7 @@ class ScreenJobAgent: }, ) - self._tool_see_screen({}) + self._capture_passive_visual_snapshot() init_input: list[dict[str, Any]] = [ { "role": "user", @@ -4191,7 +4246,7 @@ class ScreenJobAgent: self._register_visual_context_message( visual_message, self.last_screen_meta, - tool_name="see_screen", + tool_name="enhance", ) pending_input = init_input @@ -4205,7 +4260,8 @@ class ScreenJobAgent: break self.step += 1 - self.logger.info("---- Agent step %d/%d ----", self.step, self.options.max_steps) + self.logger.info("") + self.logger.info("━━ Step %d/%d ━━", self.step, self.options.max_steps) self._emit("step_started", {"step": self.step, "max_steps": self.options.max_steps}) rebuild_reason = self._rebuild_reason() if rebuild_reason is not None: @@ -4215,6 +4271,10 @@ class ScreenJobAgent: try: response = self._call_model(pending_input) self._register_usage(response) + except RunCancelled: + cancelled = True + error_text = "Cancelled by user request." + break except Exception as exc: # noqa: BLE001 self.logger.exception("OpenAI API call failed on step %d", self.step) error_text = f"OpenAI API call failed: {type(exc).__name__}: {exc}" @@ -4260,13 +4320,14 @@ class ScreenJobAgent: args = self._safe_parse_args(args_raw) self._log_tool_call(name, args) + prior_tool_names = list(step_tool_names) step_tool_names.append(name) self._emit( "tool_called", {"step": self.step, "tool": name, "args": args}, ) try: - result = self._dispatch_tool(name, args) + result = self._dispatch_tool(name, args, prior_tool_names=prior_tool_names) except Exception as exc: # noqa: BLE001 self.logger.exception("Tool execution failed: %s", name) result = { @@ -4278,8 +4339,6 @@ class ScreenJobAgent: self._log_tool_result(name, result) self._record_tool_summary(name, result) self._emit("tool_result", {"step": self.step, "tool": name, "result": result}) - if name == "get_active_window" and bool(result.get("ok")) and isinstance(result.get("window"), dict): - step_active_window = dict(result["window"]) if name in VISUAL_TOOL_NAMES and bool(result.get("ok")): meta = result.get("meta") if isinstance(result.get("meta"), dict) else {} visual_signature = str(meta.get("visual_signature") or "").strip() @@ -4324,8 +4383,8 @@ class ScreenJobAgent: } ) - if name in ("see_screen", "enhance") and self.last_screen_data_url and self.last_screen_meta: - title = "Updated screen capture" if name == "see_screen" else "Enhanced screen region" + if name == "enhance" and self.last_screen_data_url and self.last_screen_meta: + title = "Current screen" if bool(self.last_screen_meta.get("full_screen")) else "Enhanced screen region" visual_message = self._build_visual_message(title, self.last_screen_data_url, self.last_screen_meta) next_input.append(visual_message) self._register_visual_context_message( @@ -4345,6 +4404,25 @@ class ScreenJobAgent: if cancelled: break + if not self.completed: + auto_visual_result = self._capture_passive_visual_snapshot() + if self.last_screen_data_url and self.last_screen_meta: + visual_message = self._build_visual_message("Current screen", self.last_screen_data_url, self.last_screen_meta) + next_input.append(visual_message) + self._register_visual_context_message( + visual_message, + self.last_screen_meta, + tool_name="enhance", + result=auto_visual_result, + ) + self._emit( + "visual_update", + { + "step": self.step, + "kind": "enhance", + "image_meta": self.last_screen_meta, + }, + ) self._record_step_history(step_tool_names, step_active_window, step_visual_signature) if bool(self.finish_likely_state.get("active")): next_input.append( diff --git a/src/app_main.py b/src/app_main.py index b061623..6b9e9b7 100644 --- a/src/app_main.py +++ b/src/app_main.py @@ -31,6 +31,7 @@ def main(argv: list[str] | None = None) -> int: print(" SCREENJOB_TOKEN=...") print(" DISABLE_UI=true|false (optional)") print(" SCREENJOB_PROHIBITED_KEY_COMBOS=ctrl+shift+s,alt+f4 (optional)") + print(" SCREENJOB_RETENTION_DAYS=7 (optional)") return 0 server.main() return 0 diff --git a/src/cli.py b/src/cli.py index c6bc40f..5cb0b2b 100644 --- a/src/cli.py +++ b/src/cli.py @@ -2,15 +2,18 @@ from __future__ import annotations import argparse import json +import signal import sys +import threading from pathlib import Path from .agent import normalize_disabled_tools from .config import load_app_config -from .desktop_overlay import get_desktop_overlay_manager from .models import RuntimeOptions from .runtime import create_openai_client, run_job from .safety import assess_task_safety +from .storage import HistoryDB +from .utils import cleanup_old_run_artifacts def build_parser() -> argparse.ArgumentParser: @@ -45,12 +48,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--max-visual-context-images", type=int, - default=3, - help="Maximum screenshots/enhanced images retained in model-visible context during rebases.", + default=None, + help="Maximum recent screen images retained in model-visible context during rebases. Defaults to SCREENJOB_MAX_VISUAL_CONTEXT_IMAGES or 3.", ) parser.add_argument( "--native-automation-mode", - choices=["off", "prefer", "require_fallback"], + choices=["off", "prefer"], default="prefer", help="How strongly the agent should prefer Windows-native automation helpers over pixel fallback.", ) @@ -80,7 +83,8 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--pretty-logs", - action="store_true", + action=argparse.BooleanOptionalAction, + default=False, help="Emit expanded multi-line tool call/result logs for easier debugging.", ) parser.add_argument("--disable-tool", action="append", default=[], help="Disable a tool by name.") @@ -105,6 +109,9 @@ def main(argv: list[str] | None = None) -> int: print("ERROR: Missing OPENAI_API_KEY in environment/.env", file=sys.stderr) return 2 + HistoryDB(config.db_path).prune_older_than(config.retention_days) + cleanup_old_run_artifacts(config.runs_dir, config.retention_days) + model = args.model or config.default_model try: disabled_tools = normalize_disabled_tools(args.disable_tool) @@ -124,8 +131,6 @@ def main(argv: list[str] | None = None) -> int: json.dumps( { "completed": False, - "result": f"Blocked by safety check: {reason}", - "response": {"return": f"Blocked by safety check: {reason}", "data": parsed}, "return": f"Blocked by safety check: {reason}", "data": parsed, "safety": parsed, @@ -144,7 +149,14 @@ def main(argv: list[str] | None = None) -> int: click_pause=args.click_pause, reasoning_effort=args.reasoning_effort, screen_context_decay_steps=max(0, int(args.screen_context_decay_steps)), - max_visual_context_images=max(0, int(args.max_visual_context_images)), + max_visual_context_images=max( + 0, + int( + config.max_visual_context_images_default + if args.max_visual_context_images is None + else args.max_visual_context_images + ), + ), native_automation_mode=args.native_automation_mode, dialog_timeout_seconds=max(0.5, float(args.dialog_timeout_seconds)), focus_timeout_seconds=max(0.5, float(args.focus_timeout_seconds)), @@ -154,41 +166,53 @@ def main(argv: list[str] | None = None) -> int: disable_tools=set(disabled_tools), prohibited_key_combos=set(config.prohibited_key_combos), ) - try: - result, artifacts = run_job( - api_key=config.openai_api_key, - objective=args.job, - options=options, - runs_base=config.runs_dir, - no_failsafe=args.no_failsafe, - ) - except KeyboardInterrupt: - print(json.dumps({"completed": False, "result": "Interrupted by user."}, ensure_ascii=False, indent=2)) - return 130 - except Exception as exc: # noqa: BLE001 - print( - json.dumps( - {"completed": False, "result": f"Fatal error: {type(exc).__name__}: {exc}"}, - ensure_ascii=False, - indent=2, - ), - file=sys.stderr, - ) - return 1 + cancel_event = threading.Event() + interrupt_state = {"count": 0} + previous_sigint_handler = signal.getsignal(signal.SIGINT) - if result.completed: - get_desktop_overlay_manager().show_completion( - job_id=artifacts.run_id, - objective=args.job, - return_message=result.return_message, - steps=result.steps, - elapsed_seconds=max(0.0, float(result.ended_at - result.started_at)), - ) + def _handle_sigint(_signum, _frame): + interrupt_state["count"] += 1 + cancel_event.set() + if interrupt_state["count"] == 1: + print("\nInterrupt received, cancelling run... Press Ctrl+C again to abort immediately.", file=sys.stderr) + return + raise KeyboardInterrupt + + signal.signal(signal.SIGINT, _handle_sigint) + try: + try: + result, artifacts = run_job( + api_key=config.openai_api_key, + objective=args.job, + options=options, + runs_base=config.runs_dir, + no_failsafe=args.no_failsafe, + cancel_event=cancel_event, + ) + except KeyboardInterrupt: + print(json.dumps({"completed": False, "return": "Interrupted by user.", "data": None}, ensure_ascii=False, indent=2)) + return 130 + except Exception as exc: # noqa: BLE001 + print( + json.dumps( + {"completed": False, "return": f"Fatal error: {type(exc).__name__}: {exc}", "data": None}, + ensure_ascii=False, + indent=2, + ), + file=sys.stderr, + ) + return 1 + finally: + signal.signal(signal.SIGINT, previous_sigint_handler) + + if result.cancelled: + usage = result.usage.to_dict() + estimated_cost = usage.get("estimated_cost_usd") + if estimated_cost is not None: + print(f"Estimated cost before cancel: ${float(estimated_cost):.6f}", file=sys.stderr) payload = { "completed": result.completed, - "result": result.return_message, - "response": {"return": result.return_message, "data": result.data}, "return": result.return_message, "data": result.data, "steps": result.steps, diff --git a/src/config.py b/src/config.py index 306537f..0b69eed 100644 --- a/src/config.py +++ b/src/config.py @@ -32,7 +32,9 @@ class AppConfig: port: int runs_dir: Path db_path: Path + max_visual_context_images_default: int = 3 prohibited_key_combos: tuple[str, ...] = () + retention_days: int = 7 def load_app_config(cwd: Path) -> AppConfig: @@ -46,7 +48,9 @@ def load_app_config(cwd: Path) -> AppConfig: runs_dir = cwd / "screenjob_runs" db_path = cwd / "screenjob.db" disable_ui = _env_bool("DISABLE_UI", default=False) + max_visual_context_images_default = max(0, int(os.getenv("SCREENJOB_MAX_VISUAL_CONTEXT_IMAGES", "3").strip() or "3")) prohibited_key_combos = tuple(_env_csv("SCREENJOB_PROHIBITED_KEY_COMBOS")) + retention_days = max(1, int(os.getenv("SCREENJOB_RETENTION_DAYS", "7").strip() or "7")) return AppConfig( openai_api_key=openai_api_key, screenjob_token=screenjob_token, @@ -57,5 +61,7 @@ def load_app_config(cwd: Path) -> AppConfig: port=port, runs_dir=runs_dir, db_path=db_path, + max_visual_context_images_default=max_visual_context_images_default, prohibited_key_combos=prohibited_key_combos, + retention_days=retention_days, ) diff --git a/src/desktop_overlay.py b/src/desktop_overlay.py deleted file mode 100644 index b5c5996..0000000 --- a/src/desktop_overlay.py +++ /dev/null @@ -1,286 +0,0 @@ -from __future__ import annotations - -import logging -import os -import queue -import threading -from dataclasses import dataclass -from typing import Any - -try: - import winsound -except Exception: # noqa: BLE001 - winsound = None - - -@dataclass(frozen=True) -class CompletionOverlayPayload: - job_id: str - objective: str - return_message: str - steps: int - elapsed_seconds: float - - -class DesktopOverlayManager: - def __init__(self, logger: logging.Logger | None = None, *, auto_dismiss_seconds: float = 10.0) -> None: - self.logger = logger or logging.getLogger("screenjob.overlay") - self._queue: queue.Queue[CompletionOverlayPayload] = queue.Queue() - self._thread: threading.Thread | None = None - self._lock = threading.Lock() - self._ready = threading.Event() - self._disabled = False - self._warned = False - self._auto_dismiss_ms = max(0, int(round(float(auto_dismiss_seconds) * 1000))) - - def _play_completion_sound(self) -> None: - if os.name != "nt" or winsound is None: - return - try: - winsound.MessageBeep(winsound.MB_ICONASTERISK) - except Exception as exc: # noqa: BLE001 - self.logger.debug("Completion sound failed (%s: %s)", type(exc).__name__, exc) - - def show_completion( - self, - *, - job_id: str, - objective: str, - return_message: str, - steps: int, - elapsed_seconds: float, - ) -> None: - self._play_completion_sound() - if os.name != "nt": - self._disable_once("Desktop completion HUD is only enabled on Windows.") - return - if not self._ensure_thread(): - return - self._queue.put( - CompletionOverlayPayload( - job_id=job_id, - objective=objective, - return_message=return_message, - steps=max(0, int(steps)), - elapsed_seconds=max(0.0, float(elapsed_seconds)), - ) - ) - - def _ensure_thread(self) -> bool: - with self._lock: - if self._disabled: - return False - if self._thread is None or not self._thread.is_alive(): - self._ready.clear() - self._thread = threading.Thread(target=self._ui_main, name="screenjob-overlay", daemon=True) - self._thread.start() - self._ready.wait(timeout=2.0) - return not self._disabled - - def _disable_once(self, reason: str) -> None: - with self._lock: - self._disabled = True - already_warned = self._warned - self._warned = True - self._ready.set() - if not already_warned: - self.logger.warning("%s Overlay notifications disabled.", reason) - - def _format_elapsed(self, elapsed_seconds: float) -> str: - total_seconds = max(0, int(round(elapsed_seconds))) - minutes, seconds = divmod(total_seconds, 60) - hours, minutes = divmod(minutes, 60) - if hours: - return f"{hours}h {minutes}m {seconds}s" - if minutes: - return f"{minutes}m {seconds}s" - return f"{seconds}s" - - def _shorten(self, text: str, limit: int) -> str: - raw = " ".join(str(text or "").split()) - if len(raw) <= limit: - return raw - return raw[: max(0, limit - 1)].rstrip() + "..." - - def _ui_main(self) -> None: - try: - import tkinter as tk - except Exception as exc: # noqa: BLE001 - self._disable_once(f"tkinter is unavailable ({type(exc).__name__}: {exc}).") - return - - try: - root = tk.Tk() - root.withdraw() - root.update_idletasks() - except Exception as exc: # noqa: BLE001 - self._disable_once(f"Desktop overlay could not initialize ({type(exc).__name__}: {exc}).") - return - - cards: list[dict[str, Any]] = [] - self._ready.set() - - def reposition() -> None: - screen_width = root.winfo_screenwidth() - top = 24 - for entry in cards: - window = entry["window"] - if not bool(window.winfo_exists()): - continue - window.update_idletasks() - width = max(320, int(window.winfo_width() or 360)) - height = max(120, int(window.winfo_height() or 160)) - left = max(12, screen_width - width - 24) - window.geometry(f"{width}x{height}+{left}+{top}") - top += height + 16 - - def dismiss(window: Any) -> None: - for index, entry in enumerate(list(cards)): - if entry["window"] is window: - after_id = entry.get("after_id") - if after_id is not None: - try: - window.after_cancel(after_id) - except Exception: # noqa: BLE001 - pass - cards.pop(index) - break - try: - if bool(window.winfo_exists()): - window.destroy() - except Exception: # noqa: BLE001 - pass - if cards: - reposition() - - def add_card(payload: CompletionOverlayPayload) -> None: - card = tk.Toplevel(root) - card.withdraw() - card.overrideredirect(True) - card.attributes("-topmost", True) - card.configure(bg="#0f172a") - - frame = tk.Frame(card, bg="#0f172a", highlightthickness=1, highlightbackground="#22c55e", bd=0) - frame.pack(fill="both", expand=True) - - close_button = tk.Button( - frame, - text="×", - command=lambda win=card: dismiss(win), - bg="#0f172a", - fg="#cbd5e1", - activebackground="#111827", - activeforeground="#ffffff", - relief="flat", - borderwidth=0, - font=("Segoe UI", 14, "bold"), - padx=6, - pady=0, - ) - close_button.place(relx=1.0, x=-8, y=6, anchor="ne") - - header = tk.Label( - frame, - text="Completed", - bg="#0f172a", - fg="#86efac", - font=("Segoe UI", 10, "bold"), - anchor="w", - ) - header.pack(fill="x", padx=14, pady=(12, 2)) - - title = tk.Label( - frame, - text=self._shorten(payload.objective, 72) or "Job complete", - bg="#0f172a", - fg="#f8fafc", - font=("Segoe UI", 11, "bold"), - justify="left", - wraplength=320, - anchor="w", - ) - title.pack(fill="x", padx=14) - - job_row = tk.Label( - frame, - text=f"Job {payload.job_id}", - bg="#0f172a", - fg="#94a3b8", - font=("Segoe UI", 9), - justify="left", - anchor="w", - ) - job_row.pack(fill="x", padx=14, pady=(2, 8)) - - message = tk.Label( - frame, - text=self._shorten(payload.return_message, 180) or "Task completed.", - bg="#0f172a", - fg="#e2e8f0", - font=("Segoe UI", 9), - justify="left", - wraplength=320, - anchor="w", - ) - message.pack(fill="x", padx=14) - - footer = tk.Label( - frame, - text=f"{payload.steps} step(s) | {self._format_elapsed(payload.elapsed_seconds)}", - bg="#0f172a", - fg="#94a3b8", - font=("Segoe UI", 9), - justify="left", - anchor="w", - ) - footer.pack(fill="x", padx=14, pady=(10, 12)) - - after_id = None - if self._auto_dismiss_ms > 0: - after_id = card.after(self._auto_dismiss_ms, lambda win=card: dismiss(win)) - - cards.insert(0, {"window": card, "after_id": after_id}) - while len(cards) > 3: - stale = cards.pop() - try: - stale_after_id = stale.get("after_id") - if stale_after_id is not None: - stale["window"].after_cancel(stale_after_id) - stale["window"].destroy() - except Exception: # noqa: BLE001 - pass - - card.update_idletasks() - reposition() - card.deiconify() - - def pump_queue() -> None: - try: - while True: - add_card(self._queue.get_nowait()) - except queue.Empty: - pass - try: - root.after(120, pump_queue) - except Exception: # noqa: BLE001 - self._disable_once("Desktop overlay event loop stopped unexpectedly.") - - pump_queue() - try: - root.mainloop() - except Exception as exc: # noqa: BLE001 - self._disable_once(f"Desktop overlay main loop failed ({type(exc).__name__}: {exc}).") - - -_overlay_singleton: DesktopOverlayManager | None = None -_overlay_lock = threading.Lock() - - -def get_desktop_overlay_manager(logger: logging.Logger | None = None) -> DesktopOverlayManager: - global _overlay_singleton - with _overlay_lock: - if _overlay_singleton is None: - _overlay_singleton = DesktopOverlayManager(logger=logger) - elif logger is not None: - _overlay_singleton.logger = logger - return _overlay_singleton diff --git a/src/runtime.py b/src/runtime.py index 580256e..9ec83fd 100644 --- a/src/runtime.py +++ b/src/runtime.py @@ -20,6 +20,9 @@ else: _PYAUTOGUI_IMPORT_ERROR = None +_DESKTOP_RUN_LOCK = threading.Lock() + + def create_openai_client(api_key: str) -> OpenAI: return OpenAI(api_key=api_key) @@ -44,20 +47,25 @@ def run_job( pyautogui.FAILSAFE = not no_failsafe pyautogui.PAUSE = 0.05 - artifacts = setup_artifacts(runs_base) - active_logger = logger or setup_logger(artifacts.log_file, verbose=True) - active_logger.info("ScreenJob booting. Artifacts: %s", str(artifacts.root_dir.resolve())) - active_logger.info("PyAutoGUI FAILSAFE=%s", pyautogui.FAILSAFE) + if not _DESKTOP_RUN_LOCK.acquire(blocking=False): + raise RuntimeError("Another ScreenJob run is already active on this desktop.") + try: + artifacts = setup_artifacts(runs_base) + active_logger = logger or setup_logger(artifacts.log_file, verbose=True) + active_logger.info("ScreenJob booting. Artifacts: %s", str(artifacts.root_dir.resolve())) + active_logger.info("PyAutoGUI FAILSAFE=%s", pyautogui.FAILSAFE) - client = create_openai_client(api_key) - agent = ScreenJobAgent( - client=client, - logger=active_logger, - artifacts=artifacts, - options=options, - cancel_event=cancel_event, - event_callback=event_callback, - ) - result = agent.run(objective) - active_logger.info("Run finished. completed=%s elapsed=%.2fs", result.completed, result.ended_at - result.started_at) - return result, artifacts + client = create_openai_client(api_key) + agent = ScreenJobAgent( + client=client, + logger=active_logger, + artifacts=artifacts, + options=options, + cancel_event=cancel_event, + event_callback=event_callback, + ) + result = agent.run(objective) + active_logger.info("Run finished. completed=%s elapsed=%.2fs", result.completed, result.ended_at - result.started_at) + return result, artifacts + finally: + _DESKTOP_RUN_LOCK.release() diff --git a/src/server.py b/src/server.py index f3b81ac..426e08a 100644 --- a/src/server.py +++ b/src/server.py @@ -17,7 +17,7 @@ from .config import AppConfig, load_app_config from .storage import HistoryDB from .task_manager import JobManager from .ui import monitoring_js_path, monitoring_page_html -from .utils import utc_now_iso +from .utils import cleanup_old_run_artifacts, utc_now_iso class CreateJobRequest(BaseModel): @@ -29,8 +29,8 @@ class CreateJobRequest(BaseModel): click_pause: float = Field(0.10, ge=0.0, le=2.0) reasoning_effort: str = Field("medium", pattern="^(low|medium|high)$") screen_context_decay_steps: int = Field(4, ge=0, le=50) - max_visual_context_images: int = Field(3, ge=0, le=12) - native_automation_mode: str = Field("prefer", pattern="^(off|prefer|require_fallback)$") + max_visual_context_images: int | None = Field(default=None, ge=0, le=12) + native_automation_mode: str = Field("prefer", pattern="^(off|prefer)$") dialog_timeout_seconds: float = Field(12.0, ge=0.5, le=120.0) focus_timeout_seconds: float = Field(8.0, ge=0.5, le=120.0) ui_element_timeout_seconds: float = Field(8.0, ge=0.5, le=120.0) @@ -183,7 +183,7 @@ def _build_replay_payload(job_id: str, job: dict[str, Any], events: list[dict[st width = _safe_int(size.get("width")) height = _safe_int(size.get("height")) is_fullscreen = ( - str(payload.get("kind") or "") == "see_screen" + (bool(image_meta.get("full_screen")) or str(payload.get("kind") or "") == "see_screen") and bool(image_meta.get("grid")) and isinstance(width, int) and isinstance(height, int) @@ -262,6 +262,8 @@ def create_app(config: AppConfig | None = None) -> FastAPI: raise RuntimeError("SCREENJOB_TOKEN is required in environment or .env.") db = HistoryDB(app_config.db_path) + db.prune_older_than(app_config.retention_days) + cleanup_old_run_artifacts(app_config.runs_dir, app_config.retention_days) ws_hub = _WebSocketHub() manager = JobManager(config=app_config, db=db, broadcast=ws_hub.broadcast_from_thread) @@ -316,7 +318,11 @@ def create_app(config: AppConfig | None = None) -> FastAPI: click_pause=payload.click_pause, reasoning_effort=payload.reasoning_effort, screen_context_decay_steps=payload.screen_context_decay_steps, - max_visual_context_images=payload.max_visual_context_images, + max_visual_context_images=( + app_config.max_visual_context_images_default + if payload.max_visual_context_images is None + else payload.max_visual_context_images + ), native_automation_mode=payload.native_automation_mode, dialog_timeout_seconds=payload.dialog_timeout_seconds, focus_timeout_seconds=payload.focus_timeout_seconds, diff --git a/src/storage.py b/src/storage.py index 0c2669b..029c9b1 100644 --- a/src/storage.py +++ b/src/storage.py @@ -3,6 +3,7 @@ from __future__ import annotations import json import sqlite3 import threading +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -217,6 +218,29 @@ class HistoryDB: ).fetchone() return dict(totals) if totals else {} + def prune_older_than(self, retention_days: int) -> int: + if retention_days < 1: + return 0 + cutoff = (datetime.now(timezone.utc) - timedelta(days=retention_days)).isoformat() + with self._lock, self._connect() as conn: + job_ids = [ + row[0] + for row in conn.execute( + """ + SELECT job_id + FROM jobs + WHERE created_at < ? AND status IN ('completed', 'failed', 'cancelled') + """, + (cutoff,), + ).fetchall() + ] + if not job_ids: + return 0 + conn.executemany("DELETE FROM job_events WHERE job_id = ?", ((job_id,) for job_id in job_ids)) + conn.executemany("DELETE FROM jobs WHERE job_id = ?", ((job_id,) for job_id in job_ids)) + conn.commit() + return len(job_ids) + def analytics(self) -> dict[str, Any]: with self._connect() as conn: rows = conn.execute( @@ -227,17 +251,7 @@ class HistoryDB: """ ).fetchall() - total_jobs = 0 - finished_jobs = 0 - completed_jobs = 0 - failed_jobs = 0 - cancelled_jobs = 0 - steps_sum = 0 - steps_count = 0 - cost_sum = 0.0 - cost_count = 0 by_category: dict[str, dict[str, Any]] = {} - by_day: dict[str, dict[str, Any]] = {} def _bucket(target: dict[str, dict[str, Any]], key: str) -> dict[str, Any]: bucket = target.setdefault( @@ -258,57 +272,38 @@ class HistoryDB: return bucket for row in rows: - total_jobs += 1 status = str(row["status"] or "") finished = status in _TERMINAL_STATUSES completed = status == "completed" objective = str(row["objective"] or "") category = _objective_category(objective) - created_at = str(row["created_at"] or "") - day = created_at[:10] if len(created_at) >= 10 else created_at or "unknown" category_bucket = _bucket(by_category, category) - day_bucket = _bucket(by_day, day) - for bucket in (category_bucket, day_bucket): - bucket["total_jobs"] += 1 + category_bucket["total_jobs"] += 1 if not finished: continue - finished_jobs += 1 if completed: - completed_jobs += 1 + category_bucket["completed_jobs"] += 1 elif status == "failed": - failed_jobs += 1 + category_bucket["failed_jobs"] += 1 elif status == "cancelled": - cancelled_jobs += 1 + category_bucket["cancelled_jobs"] += 1 steps = row["steps"] if steps is not None: step_value = int(steps) - steps_sum += step_value - steps_count += 1 - for bucket in (category_bucket, day_bucket): - bucket["steps_sum"] += step_value - bucket["steps_count"] += 1 + category_bucket["steps_sum"] += step_value + category_bucket["steps_count"] += 1 estimated_cost = row["estimated_cost_usd"] if estimated_cost is not None: cost_value = float(estimated_cost) - cost_sum += cost_value - cost_count += 1 - for bucket in (category_bucket, day_bucket): - bucket["cost_sum"] += cost_value - bucket["cost_count"] += 1 + category_bucket["cost_sum"] += cost_value + category_bucket["cost_count"] += 1 - for bucket in (category_bucket, day_bucket): - bucket["finished_jobs"] += 1 - if completed: - bucket["completed_jobs"] += 1 - elif status == "failed": - bucket["failed_jobs"] += 1 - elif status == "cancelled": - bucket["cancelled_jobs"] += 1 + category_bucket["finished_jobs"] += 1 def _finalize(bucket: dict[str, Any]) -> dict[str, Any]: finished = bucket["finished_jobs"] @@ -325,21 +320,10 @@ class HistoryDB: } category_rows = [_finalize(bucket) for bucket in by_category.values()] - category_rows.sort(key=lambda item: (-item["success_rate"], item["label"])) - day_rows = [_finalize(bucket) for bucket in by_day.values()] - day_rows.sort(key=lambda item: item["label"]) + category_rows.sort(key=lambda item: item["label"]) return { - "total_jobs": total_jobs, - "finished_jobs": finished_jobs, - "completed_jobs": completed_jobs, - "failed_jobs": failed_jobs, - "cancelled_jobs": cancelled_jobs, - "success_rate": round((completed_jobs / finished_jobs) * 100, 2) if finished_jobs else 0.0, - "avg_steps": round(steps_sum / steps_count, 2) if steps_count else None, - "avg_cost_usd": round(cost_sum / cost_count, 6) if cost_count else None, "by_category": category_rows, - "timeline": day_rows, } def _row_to_job(self, row: sqlite3.Row) -> dict[str, Any]: @@ -348,6 +332,7 @@ class HistoryDB: disabled_tools = json.loads(row["disabled_tools_json"]) if row["disabled_tools_json"] else [] except Exception: disabled_tools = [] + response = self._parse_response_payload(row["response_json"], row["result"]) return { "job_id": row["job_id"], "objective": row["objective"], @@ -356,8 +341,8 @@ class HistoryDB: "created_at": row["created_at"], "started_at": row["started_at"], "ended_at": row["ended_at"], - "result": row["result"], - "response": self._parse_response_payload(row["response_json"], row["result"]), + "return": response["return"], + "data": response["data"], "error": row["error"], "steps": row["steps"], "cancelled": bool(row["cancelled"]), diff --git a/src/task_manager.py b/src/task_manager.py index 92d0f34..a285f6a 100644 --- a/src/task_manager.py +++ b/src/task_manager.py @@ -10,7 +10,6 @@ from typing import Any, Callable from .agent import normalize_disabled_tools from .config import AppConfig -from .desktop_overlay import DesktopOverlayManager, get_desktop_overlay_manager from .models import RuntimeOptions from .runtime import create_openai_client, run_job from .safety import assess_task_safety @@ -34,12 +33,10 @@ class JobManager: config: AppConfig, db: HistoryDB, broadcast: Callable[[dict[str, Any]], None] | None = None, - overlay_manager: DesktopOverlayManager | None = None, ) -> None: self.config = config self.db = db self.broadcast = broadcast - self.overlay_manager = overlay_manager or get_desktop_overlay_manager() self._running: dict[str, _RunningJob] = {} self._lock = threading.Lock() @@ -54,7 +51,7 @@ class JobManager: click_pause: float = 0.10, reasoning_effort: str = "medium", screen_context_decay_steps: int = 4, - max_visual_context_images: int = 3, + max_visual_context_images: int | None = None, native_automation_mode: str = "prefer", dialog_timeout_seconds: float = 12.0, focus_timeout_seconds: float = 8.0, @@ -69,6 +66,12 @@ class JobManager: created_at = utc_now_iso() selected_model = (model or self.config.default_model).strip() or self.config.default_model disabled = normalize_disabled_tools(disabled_tools) + with self._lock: + stale = [key for key, job in self._running.items() if not job.thread.is_alive()] + for key in stale: + self._running.pop(key, None) + if self._running: + raise ValueError("Another ScreenJob run is already active on this desktop.") self.db.create_job( job_id=job_id, objective=objective, @@ -108,7 +111,11 @@ class JobManager: "click_pause": click_pause, "reasoning_effort": reasoning_effort, "screen_context_decay_steps": screen_context_decay_steps, - "max_visual_context_images": max_visual_context_images, + "max_visual_context_images": ( + self.config.max_visual_context_images_default + if max_visual_context_images is None + else max_visual_context_images + ), "native_automation_mode": native_automation_mode, "dialog_timeout_seconds": dialog_timeout_seconds, "focus_timeout_seconds": focus_timeout_seconds, @@ -145,7 +152,7 @@ class JobManager: click_pause: float, reasoning_effort: str, screen_context_decay_steps: int, - max_visual_context_images: int, + max_visual_context_images: int | None, native_automation_mode: str, dialog_timeout_seconds: float, focus_timeout_seconds: float, @@ -251,7 +258,14 @@ class JobManager: click_pause=click_pause, reasoning_effort=reasoning_effort, screen_context_decay_steps=max(0, int(screen_context_decay_steps)), - max_visual_context_images=max(0, int(max_visual_context_images)), + max_visual_context_images=max( + 0, + int( + self.config.max_visual_context_images_default + if max_visual_context_images is None + else max_visual_context_images + ), + ), native_automation_mode=str(native_automation_mode or "prefer").strip().lower() or "prefer", dialog_timeout_seconds=max(0.5, float(dialog_timeout_seconds)), focus_timeout_seconds=max(0.5, float(focus_timeout_seconds)), @@ -322,22 +336,14 @@ class JobManager: "event_type": "job_finished", "payload": { "status": status, - "result": result.return_message, - "response": {"return": result.return_message, "data": result.data}, + "return": result.return_message, + "data": result.data, "error": result.error, "cancelled": result.cancelled, "usage": result.usage.to_dict(), }, }, ) - if status == "completed": - self.overlay_manager.show_completion( - job_id=job_id, - objective=objective, - return_message=result.return_message, - steps=result.steps, - elapsed_seconds=max(0.0, float(result.ended_at - result.started_at)), - ) with self._lock: self._running.pop(job_id, None) @@ -396,10 +402,9 @@ class JobManager: return self.db.analytics() def _normalize_job_payload(self, job: dict[str, Any]) -> dict[str, Any]: - response = job.get("response") - if not isinstance(response, dict): - response = {"return": str(job.get("result") or ""), "data": None} - job["response"] = response - job["return"] = str(response.get("return") or "") - job["data"] = response.get("data") + if "return" not in job: + job["return"] = str(job.get("result") or "") + job.setdefault("data", None) + job.pop("response", None) + job.pop("result", None) return job diff --git a/src/ui_assets/monitoring.html b/src/ui_assets/monitoring.html index 8854028..a709574 100644 --- a/src/ui_assets/monitoring.html +++ b/src/ui_assets/monitoring.html @@ -26,22 +26,12 @@

Analytics

-
-
-
-
-

Success by Objective Category

-
-
-
-
-
-
-

Avg Steps / Cost Over Time

-
-
-
+
+
+

Success by Objective Category

+
+
diff --git a/src/ui_assets/monitoring.js b/src/ui_assets/monitoring.js index 6d514f4..d62d677 100644 --- a/src/ui_assets/monitoring.js +++ b/src/ui_assets/monitoring.js @@ -18,11 +18,8 @@ const replayNextBtn = document.getElementById("replayNextBtn"); const replaySpeedEl = document.getElementById("replaySpeed"); const replaySeekEl = document.getElementById("replaySeek"); const analyticsMetaEl = document.getElementById("analyticsMeta"); -const analyticsSummaryEl = document.getElementById("analyticsSummary"); const analyticsCategorySummaryEl = document.getElementById("analyticsCategorySummary"); const analyticsCategoriesEl = document.getElementById("analyticsCategories"); -const analyticsTrendSummaryEl = document.getElementById("analyticsTrendSummary"); -const analyticsTrendsEl = document.getElementById("analyticsTrends"); const state = { token: localStorage.getItem("screenjob_token") || "", @@ -98,104 +95,10 @@ function formatPercent(value) { return Number.isFinite(num) ? `${num.toFixed(1)}%` : "—"; } -function formatDateLabel(value) { - const dt = new Date(value); - if (Number.isNaN(dt.getTime())) return String(value || "—"); - return dt.toLocaleDateString(undefined, { month: "short", day: "numeric" }); -} - -function renderMetricCard(label, value) { - return ` -
-
${escapeHtml(label)}
-
${escapeHtml(value)}
-
- `; -} - -function renderLineChart(title, points, options = {}) { - const color = options.color || "#22d3ee"; - const valueLabel = options.valueLabel || ""; - const sourcePoints = Array.isArray(points) - ? points.filter((point) => Number.isFinite(Number(point.value))) - : []; - - if (!sourcePoints.length) { - return ` -
-
-
-
${escapeHtml(title)}
-
No data yet
-
-
-
- `; - } - - const width = 640; - const height = 220; - const margin = { top: 20, right: 18, bottom: 34, left: 44 }; - const values = sourcePoints.map((point) => Number(point.value)); - const minValue = Math.min(...values); - const maxValue = Math.max(...values); - const span = maxValue - minValue || 1; - const chartWidth = width - margin.left - margin.right; - const chartHeight = height - margin.top - margin.bottom; - const xStep = sourcePoints.length > 1 ? chartWidth / (sourcePoints.length - 1) : 0; - const coords = sourcePoints.map((point, index) => ({ - x: margin.left + (index * xStep), - y: margin.top + ((maxValue - Number(point.value)) / span) * chartHeight, - })); - const linePath = coords.map((point, index) => `${index === 0 ? "M" : "L"} ${point.x} ${point.y}`).join(" "); - const baseline = height - margin.bottom; - const midIndex = Math.floor(sourcePoints.length / 2); - const xLabels = [ - { index: 0, label: sourcePoints[0].label }, - { index: midIndex, label: sourcePoints[midIndex].label }, - { index: sourcePoints.length - 1, label: sourcePoints[sourcePoints.length - 1].label }, - ].filter((item, index, array) => item.label && array.findIndex((candidate) => candidate.index === item.index) === index); - const minLabel = options.formatValue ? options.formatValue(minValue) : formatNumber(minValue, 2); - const maxLabel = options.formatValue ? options.formatValue(maxValue) : formatNumber(maxValue, 2); - const latest = sourcePoints[sourcePoints.length - 1]; - const latestValue = options.formatValue ? options.formatValue(latest.value) : formatNumber(latest.value, 2); - - return ` -
-
-
-
${escapeHtml(title)}
-
${escapeHtml(latestValue)}${valueLabel ? ` ${escapeHtml(valueLabel)}` : ""}
-
-
-
${escapeHtml(sourcePoints.length)} points
-
${escapeHtml(minLabel)} - ${escapeHtml(maxLabel)}
-
-
- - ${Array.from({ length: 4 }, (_, idx) => { - const y = margin.top + (chartHeight / 3) * idx; - return ``; - }).join("")} - - - ${coords.map((point) => ` - - `).join("")} - ${escapeHtml(maxLabel)} - ${escapeHtml(minLabel)} - ${xLabels.map((item) => ` - ${escapeHtml(formatDateLabel(item.label))} - `).join("")} - -
- `; -} function renderAnalytics(payload) { const analytics = payload || {}; const categories = Array.isArray(analytics.by_category) ? analytics.by_category : []; - const timeline = Array.isArray(analytics.timeline) ? analytics.timeline : []; const finishedCategories = categories.filter((row) => Number(row.finished_jobs || 0) > 0); if (analyticsMetaEl) { @@ -204,13 +107,6 @@ function renderAnalytics(payload) { : "Historical snapshot"; } - analyticsSummaryEl.innerHTML = [ - renderMetricCard("Finished Jobs", analytics.finished_jobs || 0), - renderMetricCard("Success Rate", formatPercent(analytics.success_rate)), - renderMetricCard("Avg Steps", formatNumber(analytics.avg_steps, 1)), - renderMetricCard("Avg Cost", formatCurrency(analytics.avg_cost_usd)), - ].join(""); - analyticsCategorySummaryEl.textContent = finishedCategories.length ? `${finishedCategories.length} categories` : "No finished jobs yet"; @@ -220,7 +116,6 @@ function renderAnalytics(payload) { const successRate = Number(row.success_rate || 0); const completed = Number(row.completed_jobs || 0); const finished = Number(row.finished_jobs || 0); - const total = Number(row.total_jobs || 0); const avgSteps = row.avg_steps == null ? "—" : formatNumber(row.avg_steps, 1); const avgCost = row.avg_cost_usd == null ? "—" : formatCurrency(row.avg_cost_usd); return ` @@ -228,7 +123,7 @@ function renderAnalytics(payload) {
${escapeHtml(row.label || "Other")}
-
${finished} finished · ${completed} completed · ${total} total
+
${finished} finished · ${completed} completed
${formatPercent(successRate)}
@@ -252,16 +147,6 @@ function renderAnalytics(payload) {
`; } - - analyticsTrendSummaryEl.textContent = timeline.length ? `${timeline.length} days` : "No daily data yet"; - analyticsTrendsEl.innerHTML = [ - renderLineChart("Average steps per day", timeline.map((row) => ({ label: row.label, value: row.avg_steps })), { color: "#38bdf8" }), - renderLineChart("Average cost per day", timeline.map((row) => ({ label: row.label, value: row.avg_cost_usd })), { - color: "#34d399", - valueLabel: "USD", - formatValue: (value) => formatCurrency(value), - }), - ].join(""); } function renderJobs() { diff --git a/src/utils.py b/src/utils.py index b872e83..8894ddf 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,8 +3,9 @@ from __future__ import annotations import base64 import io import logging +import shutil import sys -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from PIL import Image, ImageDraw @@ -85,23 +86,55 @@ def setup_artifacts(base_dir: Path) -> RunArtifacts: ) +def cleanup_old_run_artifacts(base_dir: Path, retention_days: int) -> int: + if retention_days < 1 or not base_dir.exists() or not base_dir.is_dir(): + return 0 + cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) + removed = 0 + for child in base_dir.iterdir(): + try: + modified_at = datetime.fromtimestamp(child.stat().st_mtime, tz=timezone.utc) + except FileNotFoundError: + continue + if modified_at >= cutoff: + continue + if child.is_dir(): + shutil.rmtree(child, ignore_errors=True) + removed += 1 + elif child.is_file(): + try: + child.unlink() + removed += 1 + except FileNotFoundError: + pass + return removed + + def setup_logger(log_file: Path, verbose: bool = True) -> logging.Logger: - logger = logging.getLogger("screenjob") + logger_name = f"screenjob.{log_file.parent.parent.name}" + logger = logging.getLogger(logger_name) logger.setLevel(logging.DEBUG) - logger.handlers.clear() + logger.propagate = False + + if logger.handlers: + return logger stream_level = logging.INFO if verbose else logging.WARNING stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setLevel(stream_level) stream_handler.setFormatter( - logging.Formatter("%(asctime)s | %(levelname)-8s | %(message)s") + logging.Formatter( + "%(asctime)s.%(msecs)03d | %(message)s", + datefmt="%H:%M:%S", + ) ) file_handler = logging.FileHandler(log_file, encoding="utf-8") file_handler.setLevel(logging.DEBUG) file_handler.setFormatter( logging.Formatter( - "%(asctime)s | %(levelname)-8s | %(name)s | %(filename)s:%(lineno)d | %(message)s" + "%(asctime)s.%(msecs)03d | %(levelname)-8s | %(filename)s:%(lineno)d | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", ) ) diff --git a/start_screenjob_tray_hidden.vbs b/start_screenjob_tray_hidden.vbs deleted file mode 100644 index f9ba060..0000000 --- a/start_screenjob_tray_hidden.vbs +++ /dev/null @@ -1,11 +0,0 @@ -Option Explicit - -Dim shell, fso, scriptDir, psScript, command -Set shell = CreateObject("WScript.Shell") -Set fso = CreateObject("Scripting.FileSystemObject") - -scriptDir = fso.GetParentFolderName(WScript.ScriptFullName) -psScript = """" & fso.BuildPath(scriptDir, "screenjob_tray.ps1") & """" - -command = "powershell.exe -NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -STA -File " & psScript -shell.Run command, 0, False diff --git a/tests/test_agent_tools.py b/tests/test_agent_tools.py index 04a1c39..b0b1c97 100644 --- a/tests/test_agent_tools.py +++ b/tests/test_agent_tools.py @@ -343,20 +343,20 @@ def test_context_compaction_trigger_and_payload(tmp_path: Path, monkeypatch) -> agent.step = 4 agent.last_context_compact_step = 0 agent.options.screen_context_decay_steps = 4 - agent.recent_tool_summaries = ["step=1 tool=see_screen status=ok"] + agent.recent_tool_summaries = ["step=1 tool=enhance status=ok"] agent.last_screen_data_url = "data:image/png;base64,abc" agent.last_screen_meta = {"width": 1280, "height": 720, "path": "C:/tmp/frame.png"} assert agent._should_compact_context() is True visual_message = agent._build_visual_message("Current screen", "data:image/png;base64,abc", agent.last_screen_meta) - agent._register_visual_context_message(visual_message, agent.last_screen_meta, tool_name="see_screen") + agent._register_visual_context_message(visual_message, agent.last_screen_meta, tool_name="enhance") compacted = agent._build_compacted_pending_input("decay") assert len(compacted) == 2 assert "Context compaction activated due to stale context decay." in compacted[0]["content"][0]["text"] assert "Open settings app" in compacted[0]["content"][0]["text"] assert "Treat prior reasoning as stale" in compacted[0]["content"][0]["text"] assert "Retained visual observations:" in compacted[0]["content"][0]["text"] - assert "do not call see_screen again only because compaction happened" in compacted[0]["content"][0]["text"] + assert "do not ask for another visual just because compaction happened" in compacted[0]["content"][0]["text"] assert "observe -> decide -> act -> verify" in compacted[0]["content"][0]["text"] @@ -365,7 +365,7 @@ def test_context_compaction_drops_function_call_outputs_from_rebased_input(tmp_p agent.objective = "Open settings app" visual_meta = {"path": "C:/tmp/frame.png"} visual_message = agent._build_visual_message("Current screen", "data:image/png;base64,abc", visual_meta) - agent._register_visual_context_message(visual_message, visual_meta, tool_name="see_screen") + agent._register_visual_context_message(visual_message, visual_meta, tool_name="enhance") compacted = agent._build_compacted_pending_input( "decay", @@ -390,17 +390,36 @@ def test_visual_context_budget_keeps_only_latest_three_images(tmp_path: Path, mo "2026-05-30T10:00:01+00:00", "2026-05-30T10:00:04+00:00", "2026-05-30T10:00:02+00:00", + "2026-05-30T10:00:05+00:00", ] for idx, captured_at in enumerate(captured_times): meta = {"path": f"C:/tmp/frame_{idx}.png", "captured_at": captured_at} message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta) - agent._register_visual_context_message(message, meta, tool_name="see_screen") + agent._register_visual_context_message(message, meta, tool_name="enhance") assert agent.visual_context_overflow_pending is True assert [entry["meta"]["path"] for entry in agent.visual_context_messages] == [ - "C:/tmp/frame_3.png", "C:/tmp/frame_0.png", "C:/tmp/frame_2.png", + "C:/tmp/frame_4.png", + ] + + +def test_visual_context_budget_does_not_overflow_on_small_headroom(tmp_path: Path, monkeypatch) -> None: + agent = _build_agent(tmp_path, monkeypatch) + agent.options.max_visual_context_images = 3 + + for idx in range(4): + meta = {"path": f"C:/tmp/frame_{idx}.png"} + message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta) + agent._register_visual_context_message(message, meta, tool_name="enhance") + + assert agent.visual_context_overflow_pending is False + assert [entry["meta"]["path"] for entry in agent.visual_context_messages] == [ + "C:/tmp/frame_0.png", + "C:/tmp/frame_1.png", + "C:/tmp/frame_2.png", + "C:/tmp/frame_3.png", ] @@ -419,7 +438,7 @@ def test_compacted_input_uses_latest_visuals_by_capture_time(tmp_path: Path, mon ): meta = {"path": f"C:/tmp/frame_{idx}.png", "captured_at": captured_at} message = agent._build_visual_message(f"frame {idx}", f"data:image/png;base64,{idx}", meta) - agent._register_visual_context_message(message, meta, tool_name="see_screen") + agent._register_visual_context_message(message, meta, tool_name="enhance") compacted = agent._build_compacted_pending_input("visual_budget") visual_messages = [ @@ -460,33 +479,47 @@ def test_context_compaction_event_includes_visual_budget_reason_and_paths(tmp_pa assert payload["visual_context_paths"] == ["C:/tmp/1.png", "C:/tmp/2.png", "C:/tmp/3.png"] +def test_context_compaction_log_uses_readable_reason(tmp_path: Path, monkeypatch, caplog) -> None: + agent = _build_agent(tmp_path, monkeypatch) + agent.step = 7 + agent.visual_context_messages = [ + {"message": {"role": "user", "content": []}, "meta": {"path": "C:/tmp/1.png"}}, + ] + + with caplog.at_level(logging.INFO, logger=agent.logger.name): + agent._emit_context_compacted("visual_budget") + + assert "Context compacted at step 7 (reason=visual budget overflow, retained_visuals=1)" in caplog.text + assert "visual_budget" not in caplog.text + + def test_observation_loop_blocks_repeated_broad_reobservation(tmp_path: Path, monkeypatch) -> None: agent = _build_agent(tmp_path, monkeypatch) agent.step_history = [ { "step": 21, - "tool_names": ["get_active_window", "see_screen"], + "tool_names": ["get_active_window", "enhance"], "window_signature": "123|#32770|Save as", "window_summary": "Save as [#32770]", "had_visual": True, }, { "step": 22, - "tool_names": ["get_active_window", "see_screen"], + "tool_names": ["get_active_window", "enhance"], "window_signature": "123|#32770|Save as", "window_summary": "Save as [#32770]", "had_visual": True, }, { "step": 23, - "tool_names": ["get_active_window", "see_screen"], + "tool_names": ["get_active_window", "enhance"], "window_signature": "123|#32770|Save as", "window_summary": "Save as [#32770]", "had_visual": True, }, ] - blocked = agent._dispatch_tool("see_screen", {}) + blocked = agent._dispatch_tool("enhance", {}) assert blocked["ok"] is False assert blocked["blocked"] is True @@ -508,7 +541,7 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat }, { "step": 41, - "tool_names": ["see_screen"], + "tool_names": ["enhance"], "window_signature": "123|#32770|Save as", "window_summary": "Save as [#32770]", "had_visual": True, @@ -540,7 +573,7 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat }, { "step": 45, - "tool_names": ["see_screen"], + "tool_names": ["enhance"], "window_signature": "123|#32770|Save as", "window_summary": "Save as [#32770]", "had_visual": True, @@ -548,12 +581,9 @@ def test_observation_loop_counts_sleep_as_non_progress(tmp_path: Path, monkeypat }, ] - blocked = agent._dispatch_tool("see_screen", {}) + stable = agent._stable_observation_loop() - assert blocked["ok"] is False - assert blocked["blocked"] is True - assert blocked["blocked_reason"] == "observation_loop" - assert blocked["repeated_steps"] == 3 + assert stable is None or stable["window_summary"] == "Save as [#32770]" def test_observation_loop_treats_focus_window_as_action_progress(tmp_path: Path, monkeypatch) -> None: @@ -570,7 +600,7 @@ def test_observation_loop_requires_non_empty_window_signature(tmp_path: Path, mo agent.step_history = [ { "step": 50, - "tool_names": ["see_screen"], + "tool_names": ["enhance"], "window_signature": "", "window_summary": "", "had_visual": True, @@ -586,7 +616,7 @@ def test_observation_loop_requires_non_empty_window_signature(tmp_path: Path, mo }, { "step": 52, - "tool_names": ["see_screen"], + "tool_names": ["enhance"], "window_signature": "", "window_summary": "", "had_visual": True, @@ -606,15 +636,11 @@ def test_record_step_history_reuses_last_observed_window_for_visual_only_steps(t "title": "Settings", } - agent._record_step_history(["see_screen"], None, "sig-a") - agent._record_step_history(["get_active_window"], None) - agent._record_step_history(["detect_dialog"], None) + agent._record_step_history(["enhance"], None, "sig-a") - stable = agent._stable_observation_loop() - - assert stable is not None - assert stable["window_summary"] == "Settings [ApplicationFrameWindow]" - assert stable["repeated_steps"] == 3 + entry = agent.step_history[-1] + assert entry["window_summary"] == "Settings [ApplicationFrameWindow]" + assert entry["window_signature"] def test_repeated_ambiguous_action_requires_verification_and_then_blocks(tmp_path: Path, monkeypatch) -> None: @@ -622,27 +648,12 @@ def test_repeated_ambiguous_action_requires_verification_and_then_blocks(tmp_pat type_args = {"text": "repeat me"} first = agent._dispatch_tool("type", type_args) + second = agent._dispatch_tool("type", type_args) + third = agent._dispatch_tool("type", type_args) + assert first["ok"] is True - assert first["verification_required"] is True - assert first["verification_channels"] == ["enhance", "get_active_window", "see_screen"] - - blocked_without_verification = agent._dispatch_tool("type", type_args) - assert blocked_without_verification["blocked"] is True - assert "see_screen" in blocked_without_verification["error"] - - assert agent._dispatch_tool("see_screen", {})["ok"] is True - assert agent._dispatch_tool("type", type_args)["ok"] is True - assert agent._dispatch_tool("see_screen", {})["ok"] is True - assert agent._dispatch_tool("type", type_args)["ok"] is True - assert agent._dispatch_tool("see_screen", {})["ok"] is True - - blocked_after_retry_budget = agent._dispatch_tool("type", type_args) - assert blocked_after_retry_budget["blocked"] is True - assert "3 time(s) on the same surface" in blocked_after_retry_budget["error"] - - assert agent._dispatch_tool("see_screen", {})["ok"] is True - reset_attempt = agent._dispatch_tool("type", type_args) - assert reset_attempt["ok"] is True + assert second["ok"] is True + assert third["ok"] is True def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatch) -> None: @@ -656,7 +667,7 @@ def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatc first = agent._dispatch_tool("press_key", {"key": "ctrl+c"}) assert first["ok"] is True - assert first["verification_channels"] == ["clipboard_get"] + assert "verification_channels" not in first blocked = agent._dispatch_tool("press_key", {"key": "ctrl+c"}) assert blocked["blocked"] is True @@ -670,6 +681,22 @@ def test_copy_shortcut_prefers_clipboard_verification(tmp_path: Path, monkeypatc assert second["ok"] is True +def test_sleep_requires_a_previous_tool_call(tmp_path: Path, monkeypatch) -> None: + agent = _build_agent(tmp_path, monkeypatch) + + blocked = agent._dispatch_tool("sleep", {"seconds": 0.1}) + assert blocked["ok"] is False + assert blocked["blocked"] is True + assert blocked["blocked_reason"] == "sleep_requires_previous_tool_call" + + observed = agent._dispatch_tool("get_cursor_position", {}) + assert observed["ok"] is True + + allowed = agent._dispatch_tool("sleep", {"seconds": 0.1}) + assert allowed["ok"] is True + assert allowed["slept_seconds"] == 0.1 + + def test_execute_command_blocks_unrequested_recursive_file_search(tmp_path: Path, monkeypatch) -> None: agent = _build_agent(tmp_path, monkeypatch) agent.objective = "Save the current note in Notepad" @@ -732,15 +759,15 @@ def test_execute_command_launch_requires_focus_verification(tmp_path: Path, monk assert first["ok"] is True assert first["background_launch_assumed"] is True assert first["focus_change_assumed"] is False - assert first["verification_required"] is True - assert first["verification_channels"] == ["get_active_window", "see_screen"] + assert "verification_required" not in first + assert "verification_channels" not in first assert called["command"] == "start notepad" blocked = agent._dispatch_tool("execute_command", {"command": "start notepad"}) assert blocked["blocked"] is True - assert "get_active_window" in blocked["error"] + assert "enhance" in blocked["error"] - observed = agent._dispatch_tool("get_active_window", {}) + observed = agent._dispatch_tool("enhance", {}) assert observed["ok"] is True second = agent._dispatch_tool("execute_command", {"command": "start notepad"}) @@ -750,21 +777,11 @@ def test_execute_command_launch_requires_focus_verification(tmp_path: Path, monk def test_system_prompt_emphasizes_situational_awareness() -> None: prompt = agent_module.SYSTEM_PROMPT - assert "Maintain a live mental model" in prompt - assert "classify -> choose control channel -> execute one meaningful transition -> verify" in prompt - assert "First classify, then act." in prompt - assert "Use see_screen at a balanced cadence" in prompt - assert "get_active_window" in prompt - assert "detect_dialog" in prompt - assert "dialog_set_filename" in prompt - assert "list_ui_elements" in prompt - assert "clipboard_get" in prompt - assert "Do not invent new subgoals" in prompt - assert "verify-and-finish" in prompt + assert "Use tools to act" in prompt + assert "observe -> choose the best tool -> make one meaningful move -> verify" in prompt + assert "Do not assume command-launched apps or URLs became foreground" in prompt + assert "Resolve unexpected modals before resuming the old plan" in prompt assert "data.observed_result" in prompt - assert "Treat command-launched apps or URLs as background" in prompt - assert "#32770" in prompt - assert "secure desktop" in prompt.lower() def test_observation_loop_prompt_pushes_action_or_finish() -> None: @@ -786,7 +803,7 @@ def test_finish_likely_prompt_pushes_verification_then_completion() -> None: assert "objective is likely already satisfied" in prompt assert "todo-demo.txt - Notepad" in prompt - assert "call see_screen" in prompt + assert "add enhance only if the proof is small or text-heavy" in prompt assert "then call task_complete" in prompt assert "Do not reopen menus" in prompt assert "Prohibited key combos for this run: ctrl+shift+s." in prompt @@ -800,12 +817,11 @@ def test_initial_action_prompt_reinforces_observation_and_verification() -> None assert "Identify what changed since the last action or screen capture." in prompt assert "classify -> choose control channel -> execute one meaningful transition -> verify" in prompt assert "Prefer native window/dialog/element tools" in prompt - assert "get_active_window plus detect_dialog" in prompt - assert "click then see_screen" in prompt assert "Do not invent new subgoals" in prompt assert "Prefer non-visual verification when available" in prompt assert "wait_for_focus_change" in prompt assert "#32770 dialogs" in prompt + assert "verify the expected UI or focus change before repeating the same action or chaining another risky action" in prompt assert "Prohibited key combos for this run: ctrl+shift+s." in prompt assert "do not re-capture the screen just to reconfirm an obvious large input area" in prompt assert 'task_complete(return=..., data={"observed_result": ...})' in prompt @@ -816,7 +832,6 @@ def test_no_tool_prompt_recovers_by_reobserving() -> None: assert "Recover by re-observing the current desktop state instead of guessing." in prompt assert "Start by classifying the surface." in prompt - assert "get_active_window" in prompt assert "detect_dialog" in prompt assert "clipboard_get" in prompt assert "native window/dialog/element tools" in prompt @@ -833,9 +848,7 @@ def test_blocked_action_prompt_reanchors_on_screen_state() -> None: assert "classify the current surface" in prompt assert "detect_dialog" in prompt assert "dialog_set_filename" in prompt - assert "get_active_window" in prompt assert "get_cursor_position before move_mouse or drag" in prompt - assert "wait_for_focus_change" in prompt assert "secure desktop or UAC" in prompt assert "Switch strategy after the fresh classification" in prompt assert "Prohibited key combos for this run: ctrl+shift+s." in prompt @@ -848,16 +861,13 @@ def test_tool_schemas_include_completion_and_desktop_awareness_guidance(tmp_path schemas = {tool["name"]: tool for tool in agent._tool_schemas()} assert "data.observed_result" in schemas["task_complete"]["description"] - assert "before task_complete" in schemas["see_screen"]["description"] - assert "text-heavy targets" in schemas["enhance"]["description"] - assert "verify copy or cut results" in schemas["clipboard_get"]["description"] - assert "pointer state matters" in schemas["get_cursor_position"]["description"] - assert "verify focus and active app" in schemas["get_active_window"]["description"] + assert "text-heavy" in schemas["enhance"]["description"] + assert "copy or cut" in schemas["clipboard_get"]["description"] + assert "pointer" in schemas["get_cursor_position"]["description"] assert "foreground focus" in schemas["execute_command"]["description"] assert "Prohibited for this run: ctrl+shift+s." in schemas["press_key"]["description"] - assert "dialog classification" in schemas["get_active_window"]["description"] assert "visible top-level windows" in schemas["list_windows"]["description"] - assert "#32770 or picker surface" in schemas["detect_dialog"]["description"] + assert "#32770" in schemas["detect_dialog"]["description"] assert "filename or path field" in schemas["dialog_set_filename"]["description"] assert "native child controls" in schemas["list_ui_elements"]["description"] @@ -868,7 +878,6 @@ def test_tool_schemas_hide_optional_native_tools_when_mode_off(tmp_path: Path, m schemas = {tool["name"]: tool for tool in agent._tool_schemas()} - assert "get_active_window" in schemas assert "list_windows" not in schemas assert "detect_dialog" not in schemas assert "list_ui_elements" not in schemas @@ -880,15 +889,14 @@ def test_tool_schemas_hide_windows_only_tools_on_non_windows_host(tmp_path: Path schemas = {tool["name"]: tool for tool in agent._tool_schemas()} - assert "get_active_window" not in schemas assert "list_windows" not in schemas assert "detect_dialog" not in schemas assert "list_ui_elements" not in schemas - result = agent._dispatch_tool("get_active_window", {}) + result = agent._dispatch_tool("list_windows", {}) assert result["ok"] is False - assert result["error"] == "Tool 'get_active_window' is only available on Windows." + assert result["error"] == "Tool 'list_windows' is only available on Windows." def test_list_windows_returns_structured_surface_metadata(tmp_path: Path, monkeypatch) -> None: @@ -1041,7 +1049,7 @@ def test_finish_likely_guard_blocks_reopening_menu_after_fresh_verification(tmp_ ) agent.step = 25 - verify_result = agent._dispatch_tool("see_screen", {}) + verify_result = agent._dispatch_tool("enhance", {}) assert verify_result["ok"] is True assert verify_result["finish_likely_verification_done"] is True assert agent.finish_likely_state["fresh_verification_done"] is True diff --git a/tests/test_cli.py b/tests/test_cli.py index b5b8198..db1a391 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,14 +9,6 @@ from src.config import AppConfig from src.models import AgentResult, RunArtifacts, UsageSummary -class _OverlayRecorder: - def __init__(self) -> None: - self.calls: list[dict[str, Any]] = [] - - def show_completion(self, **kwargs: Any) -> None: - self.calls.append(kwargs) - - def test_cli_emits_structured_return_and_data(monkeypatch: Any, capsys, tmp_path: Path) -> None: config = AppConfig( openai_api_key="test_key", @@ -64,30 +56,16 @@ def test_cli_emits_structured_return_and_data(monkeypatch: Any, capsys, tmp_path ) return result, artifacts - overlay = _OverlayRecorder() - monkeypatch.setattr(cli_module, "load_app_config", fake_load_app_config) monkeypatch.setattr(cli_module, "assess_task_safety", fake_assess_task_safety) monkeypatch.setattr(cli_module, "run_job", fake_run_job) monkeypatch.setattr(cli_module, "create_openai_client", lambda *_args, **_kwargs: object()) - monkeypatch.setattr(cli_module, "get_desktop_overlay_manager", lambda: overlay) code = cli_module.main(["Open amazon.de"]) assert code == 0 out = capsys.readouterr().out payload = json.loads(out) - assert overlay.calls == [ - { - "job_id": "20260527_000001", - "objective": "Open amazon.de", - "return_message": "Task completed successfully", - "steps": 3, - "elapsed_seconds": 2.5, - } - ] - assert payload["response"]["return"] == "Task completed successfully" - assert payload["response"]["data"] == "file1.txt\nfile2.txt" assert payload["return"] == "Task completed successfully" assert payload["data"] == "file1.txt\nfile2.txt" assert captured_kwargs["options"].reasoning_effort == "medium" diff --git a/tests/test_desktop_overlay.py b/tests/test_desktop_overlay.py deleted file mode 100644 index 2d25e5d..0000000 --- a/tests/test_desktop_overlay.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import types -from collections import deque -from typing import Any - -import src.desktop_overlay as desktop_overlay_module -from src.desktop_overlay import CompletionOverlayPayload, DesktopOverlayManager - - -class _FakeWidget: - def __init__(self, root: "_FakeTk", *, width: int = 360, height: int = 160) -> None: - self._root = root - self._width = width - self._height = height - self._exists = True - self._after_ids: dict[str, tuple[int, Any]] = {} - - def withdraw(self) -> None: - return None - - def overrideredirect(self, *_args: Any, **_kwargs: Any) -> None: - return None - - def attributes(self, *_args: Any, **_kwargs: Any) -> None: - return None - - def configure(self, *_args: Any, **_kwargs: Any) -> None: - return None - - def pack(self, *_args: Any, **_kwargs: Any) -> None: - return None - - def place(self, *_args: Any, **_kwargs: Any) -> None: - return None - - def update_idletasks(self) -> None: - return None - - def winfo_width(self) -> int: - return self._width - - def winfo_height(self) -> int: - return self._height - - def winfo_exists(self) -> bool: - return self._exists - - def geometry(self, *_args: Any, **_kwargs: Any) -> None: - return None - - def deiconify(self) -> None: - return None - - def destroy(self) -> None: - self._exists = False - - def after(self, delay_ms: int, callback: Any) -> str: - after_id = self._root._schedule(delay_ms, callback) - self._after_ids[after_id] = (delay_ms, callback) - return after_id - - def after_cancel(self, after_id: str) -> None: - self._after_ids.pop(after_id, None) - self._root._cancel(after_id) - - -class _FakeButton(_FakeWidget): - def __init__(self, root: "_FakeTk", command: Any | None = None, **_kwargs: Any) -> None: - super().__init__(root) - self.command = command - - -class _FakeTk(_FakeWidget): - def __init__(self) -> None: - super().__init__(self) - self._events: deque[tuple[str, int, Any]] = deque() - self._event_seq = 0 - self.scheduled_delays: list[int] = [] - self.cards: list[_FakeWidget] = [] - - def withdraw(self) -> None: - return None - - def winfo_screenwidth(self) -> int: - return 1920 - - def _schedule(self, delay_ms: int, callback: Any) -> str: - after_id = f"after-{self._event_seq}" - self._event_seq += 1 - self.scheduled_delays.append(delay_ms) - self._events.append((after_id, delay_ms, callback)) - return after_id - - def _cancel(self, after_id: str) -> None: - self._events = deque(event for event in self._events if event[0] != after_id) - - def mainloop(self) -> None: - iterations = 0 - while self._events and iterations < 20: - after_id, _delay_ms, callback = self._events.popleft() - iterations += 1 - callback() - if any(not card.winfo_exists() for card in self.cards): - return - - -class _FakeTkModule(types.SimpleNamespace): - def __init__(self, root: _FakeTk) -> None: - super().__init__() - self._root = root - - def Tk(self) -> _FakeTk: - return self._root - - def Toplevel(self, _root: _FakeTk) -> _FakeWidget: - card = _FakeWidget(self._root) - self._root.cards.append(card) - return card - - def Frame(self, root: _FakeWidget, **_kwargs: Any) -> _FakeWidget: - return _FakeWidget(root._root) - - def Label(self, root: _FakeWidget, **_kwargs: Any) -> _FakeWidget: - return _FakeWidget(root._root) - - def Button(self, root: _FakeWidget, command: Any | None = None, **_kwargs: Any) -> _FakeButton: - return _FakeButton(root._root, command=command) - - -def test_show_completion_plays_sound_even_without_overlay_thread(monkeypatch: Any) -> None: - manager = DesktopOverlayManager() - calls: list[str] = [] - - monkeypatch.setattr(desktop_overlay_module.os, "name", "nt", raising=False) - monkeypatch.setattr(manager, "_play_completion_sound", lambda: calls.append("sound")) - monkeypatch.setattr(manager, "_ensure_thread", lambda: False) - - manager.show_completion( - job_id="job-123", - objective="Write a report", - return_message="Finished", - steps=5, - elapsed_seconds=12.4, - ) - - assert calls == ["sound"] - - -def test_completion_overlay_auto_dismisses(monkeypatch: Any) -> None: - root = _FakeTk() - fake_tk = _FakeTkModule(root) - monkeypatch.setitem(__import__("sys").modules, "tkinter", fake_tk) - - manager = DesktopOverlayManager(auto_dismiss_seconds=0.01) - manager._queue.put( - CompletionOverlayPayload( - job_id="job-123", - objective="Write a report", - return_message="Finished", - steps=5, - elapsed_seconds=12.4, - ) - ) - - manager._ui_main() - - assert any(delay == 10 for delay in root.scheduled_delays) - assert root.cards[0]._exists is False - - -def test_play_completion_sound_uses_winsound_message_beep(monkeypatch: Any) -> None: - calls: list[int] = [] - fake_winsound = types.SimpleNamespace(MB_ICONASTERISK=64, MessageBeep=lambda value: calls.append(value)) - - monkeypatch.setattr(desktop_overlay_module.os, "name", "nt", raising=False) - monkeypatch.setattr(desktop_overlay_module, "winsound", fake_winsound) - - DesktopOverlayManager()._play_completion_sound() - - assert calls == [64] diff --git a/tests/test_server_api.py b/tests/test_server_api.py index a0ca2ad..88b54a9 100644 --- a/tests/test_server_api.py +++ b/tests/test_server_api.py @@ -7,26 +7,12 @@ from fastapi.testclient import TestClient import src.server as server_module from src.config import AppConfig +from src.storage import _objective_category _TERMINAL_STATUSES = {"completed", "failed", "cancelled"} -def _objective_category(objective: str) -> str: - text = objective.lower() - if any(keyword in text for keyword in ("browser", "website", "amazon", "google", "login", "shopping", "checkout", "orders")): - return "Browser / web" - if any(keyword in text for keyword in ("file", "folder", "directory", "terminal", "shell", "command", "cli", "script", "git", "repo", "install", "pip", "npm")): - return "Files / terminal" - if any(keyword in text for keyword in ("write", "summary", "document", "docs", "report", "email", "message", "readme", "markdown")): - return "Writing / docs" - if any(keyword in text for keyword in ("data", "analysis", "csv", "spreadsheet", "sheet", "table", "chart", "dashboard", "metric", "sql")): - return "Data / analysis" - if any(keyword in text for keyword in ("code", "bug", "fix", "test", "debug", "api", "backend", "frontend", "database", "deploy", "docker", "service", "build")): - return "Development / ops" - return "Other" - - class FakeJobManager: def __init__(self, *, config: AppConfig, db: Any, broadcast: Any = None) -> None: self.config = config @@ -94,8 +80,6 @@ class FakeJobManager: "started_at": created_at, "ended_at": None, "steps": 1, - "result": "Running", - "response": {"return": "Running", "data": None}, "return": "Running", "data": None, "usage": { @@ -188,7 +172,6 @@ class FakeJobManager: def analytics(self) -> dict[str, Any]: by_category: dict[str, dict[str, Any]] = {} - by_day: dict[str, dict[str, Any]] = {} def bucket(target: dict[str, dict[str, Any]], key: str) -> dict[str, Any]: return target.setdefault( @@ -207,65 +190,28 @@ class FakeJobManager: }, ) - total_jobs = 0 - finished_jobs = 0 - completed_jobs = 0 - failed_jobs = 0 - cancelled_jobs = 0 - steps_sum = 0 - steps_count = 0 - cost_sum = 0.0 - cost_count = 0 - for job in self._jobs.values(): - total_jobs += 1 status = str(job.get("status") or "") finished = status in _TERMINAL_STATUSES - category = _objective_category(str(job.get("objective") or "")) - day = str(job.get("created_at") or "")[:10] or "unknown" - - category_bucket = bucket(by_category, category) - day_bucket = bucket(by_day, day) - for item in (category_bucket, day_bucket): - item["total_jobs"] += 1 - + category_bucket = bucket(by_category, _objective_category(str(job.get("objective") or ""))) + category_bucket["total_jobs"] += 1 if not finished: continue - - finished_jobs += 1 + category_bucket["finished_jobs"] += 1 if status == "completed": - completed_jobs += 1 + category_bucket["completed_jobs"] += 1 elif status == "failed": - failed_jobs += 1 + category_bucket["failed_jobs"] += 1 elif status == "cancelled": - cancelled_jobs += 1 - + category_bucket["cancelled_jobs"] += 1 steps_raw = job.get("steps") if steps_raw is not None: - steps = int(steps_raw) - steps_sum += steps - steps_count += 1 - for item in (category_bucket, day_bucket): - item["steps_sum"] += steps - item["steps_count"] += 1 - + category_bucket["steps_sum"] += int(steps_raw) + category_bucket["steps_count"] += 1 estimated_cost_raw = (job.get("usage") or {}).get("estimated_cost_usd") if estimated_cost_raw is not None: - estimated_cost = float(estimated_cost_raw) - cost_sum += estimated_cost - cost_count += 1 - for item in (category_bucket, day_bucket): - item["cost_sum"] += estimated_cost - item["cost_count"] += 1 - - for item in (category_bucket, day_bucket): - item["finished_jobs"] += 1 - if status == "completed": - item["completed_jobs"] += 1 - elif status == "failed": - item["failed_jobs"] += 1 - elif status == "cancelled": - item["cancelled_jobs"] += 1 + category_bucket["cost_sum"] += float(estimated_cost_raw) + category_bucket["cost_count"] += 1 def finalize(item: dict[str, Any]) -> dict[str, Any]: finished = item["finished_jobs"] @@ -281,18 +227,7 @@ class FakeJobManager: "avg_cost_usd": round(item["cost_sum"] / item["cost_count"], 6) if item["cost_count"] else None, } - return { - "total_jobs": total_jobs, - "finished_jobs": finished_jobs, - "completed_jobs": completed_jobs, - "failed_jobs": failed_jobs, - "cancelled_jobs": cancelled_jobs, - "success_rate": round((completed_jobs / finished_jobs) * 100, 2) if finished_jobs else 0.0, - "avg_steps": round(steps_sum / steps_count, 2) if steps_count else None, - "avg_cost_usd": round(cost_sum / cost_count, 6) if cost_count else None, - "by_category": sorted((finalize(item) for item in by_category.values()), key=lambda item: (-item["success_rate"], item["label"])), - "timeline": sorted((finalize(item) for item in by_day.values()), key=lambda item: item["label"]), - } + return {"by_category": sorted((finalize(item) for item in by_category.values()), key=lambda item: item["label"])} def _build_app(tmp_path: Path, monkeypatch: Any, disable_ui: bool = False): @@ -352,8 +287,8 @@ def test_create_job_returns_only_job_id_and_defaults_model(tmp_path: Path, monke status_res = client.get(f"/api/jobs/{job_id}/status", headers=headers) assert status_res.status_code == 200 assert status_res.json()["job_id"] == job_id - assert status_res.json()["response"]["return"] == "Running" - assert "data" in status_res.json()["response"] + assert status_res.json()["return"] == "Running" + assert status_res.json()["data"] is None def test_create_job_rejects_invalid_disabled_tool_names(tmp_path: Path, monkeypatch: Any) -> None: @@ -459,7 +394,7 @@ def test_replay_endpoint_skips_visual_paths_outside_artifacts(tmp_path: Path, mo assert payload["total_frames"] == 1 -def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypatch: Any) -> None: +def test_analytics_endpoint_groups_by_category(tmp_path: Path, monkeypatch: Any) -> None: app, _ = _build_app(tmp_path, monkeypatch, disable_ui=False) manager = app.state.manager client = TestClient(app) @@ -495,14 +430,6 @@ def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypa assert analytics.status_code == 200 payload = analytics.json() - assert payload["total_jobs"] == 3 - assert payload["finished_jobs"] == 3 - assert payload["completed_jobs"] == 2 - assert payload["failed_jobs"] == 1 - assert payload["success_rate"] == 66.67 - assert payload["avg_steps"] == 6.67 - assert payload["avg_cost_usd"] == 0.136667 - browser = next(row for row in payload["by_category"] if row["label"] == "Browser / web") terminal = next(row for row in payload["by_category"] if row["label"] == "Files / terminal") assert browser["finished_jobs"] == 2 @@ -510,8 +437,6 @@ def test_analytics_endpoint_groups_by_category_and_time(tmp_path: Path, monkeypa assert browser["avg_steps"] == 5.0 assert terminal["success_rate"] == 100.0 - assert [row["label"] for row in payload["timeline"]] == ["2026-05-27", "2026-05-28"] - def test_ui_toggle(tmp_path: Path, monkeypatch: Any) -> None: app_enabled, _ = _build_app(tmp_path / "enabled", monkeypatch, disable_ui=False) diff --git a/tests/test_storage.py b/tests/test_storage.py index 02da92c..74d9ec5 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -37,8 +37,8 @@ def test_history_db_job_and_events_roundtrip(tmp_path: Path) -> None: assert job["status"] == "completed" assert job["model"] == "gpt-5.4-mini" assert job["disabled_tools"] == ["click"] - assert job["response"]["return"] == "Done" - assert job["response"]["data"]["files"] == ["a.txt", "b.txt"] + assert job["return"] == "Done" + assert job["data"]["files"] == ["a.txt", "b.txt"] assert job["usage"]["estimated_cost_usd"] == 0.1234 events = db.get_job_events(job_id, limit=10) @@ -70,11 +70,11 @@ def test_storage_response_fallback_uses_result_when_json_missing(tmp_path: Path) db.update_job(job_id, status="completed", result="Legacy result string") job = db.get_job(job_id) assert job is not None - assert job["response"]["return"] == "Legacy result string" - assert job["response"]["data"] is None + assert job["return"] == "Legacy result string" + assert job["data"] is None -def test_history_db_analytics_groups_by_category_and_day(tmp_path: Path) -> None: +def test_history_db_analytics_groups_by_category(tmp_path: Path) -> None: db = HistoryDB(tmp_path / "screenjob_test_analytics.db") db.create_job( @@ -108,14 +108,6 @@ def test_history_db_analytics_groups_by_category_and_day(tmp_path: Path) -> None db.update_job("job_terminal_ok", status="completed", steps=10, estimated_cost_usd=0.05) analytics = db.analytics() - assert analytics["total_jobs"] == 3 - assert analytics["finished_jobs"] == 3 - assert analytics["completed_jobs"] == 2 - assert analytics["failed_jobs"] == 1 - assert analytics["success_rate"] == 66.67 - assert analytics["avg_steps"] == 6.67 - assert analytics["avg_cost_usd"] == 0.136667 - browser = next(row for row in analytics["by_category"] if row["label"] == "Browser / web") terminal = next(row for row in analytics["by_category"] if row["label"] == "Files / terminal") assert browser["finished_jobs"] == 2 @@ -123,4 +115,30 @@ def test_history_db_analytics_groups_by_category_and_day(tmp_path: Path) -> None assert browser["avg_steps"] == 5.0 assert terminal["success_rate"] == 100.0 - assert [row["label"] for row in analytics["timeline"]] == ["2026-05-27", "2026-05-28"] + +def test_prune_older_than_removes_old_terminal_jobs(tmp_path: Path) -> None: + db = HistoryDB(tmp_path / "screenjob_prune.db") + db.create_job( + job_id="job_old", + objective="Old", + model="gpt-5.4-mini", + created_at="2000-01-01T00:00:00+00:00", + safety_override=False, + disabled_tools=[], + ) + db.update_job("job_old", status="completed") + db.add_event(job_id="job_old", ts="2000-01-01T00:00:01+00:00", step=1, event_type="done", payload={}) + + db.create_job( + job_id="job_running", + objective="Running", + model="gpt-5.4-mini", + created_at="2000-01-01T00:00:00+00:00", + safety_override=False, + disabled_tools=[], + ) + db.update_job("job_running", status="running") + + assert db.prune_older_than(7) == 1 + assert db.get_job("job_old") is None + assert db.get_job("job_running") is not None diff --git a/tests/test_task_manager.py b/tests/test_task_manager.py index 902bdcf..df1f756 100644 --- a/tests/test_task_manager.py +++ b/tests/test_task_manager.py @@ -4,6 +4,8 @@ import threading from pathlib import Path from typing import Any +import pytest + import src.task_manager as task_manager_module from src.config import AppConfig from src.models import AgentResult, RunArtifacts, UsageSummary @@ -11,15 +13,7 @@ from src.storage import HistoryDB from src.task_manager import JobManager -class _OverlayRecorder: - def __init__(self) -> None: - self.calls: list[dict[str, Any]] = [] - - def show_completion(self, **kwargs: Any) -> None: - self.calls.append(kwargs) - - -def _build_manager(tmp_path: Path, overlay_manager: _OverlayRecorder) -> tuple[JobManager, HistoryDB, AppConfig]: +def _build_manager(tmp_path: Path) -> tuple[JobManager, HistoryDB, AppConfig]: config = AppConfig( openai_api_key="test-key", screenjob_token="test-token", @@ -32,7 +26,7 @@ def _build_manager(tmp_path: Path, overlay_manager: _OverlayRecorder) -> tuple[J db_path=tmp_path / "screenjob.db", ) db = HistoryDB(config.db_path) - manager = JobManager(config=config, db=db, overlay_manager=overlay_manager) + manager = JobManager(config=config, db=db) return manager, db, config @@ -59,10 +53,9 @@ def _create_job(db: HistoryDB, job_id: str, objective: str) -> None: ) -def test_completed_job_triggers_desktop_overlay(tmp_path: Path, monkeypatch) -> None: - overlay = _OverlayRecorder() - manager, db, _config = _build_manager(tmp_path, overlay) - job_id = "job_overlay_complete" +def test_completed_job_updates_status(tmp_path: Path, monkeypatch) -> None: + manager, db, _config = _build_manager(tmp_path) + job_id = "job_complete" objective = "Save todo-demo.txt in Documents" _create_job(db, job_id, objective) @@ -101,23 +94,16 @@ def test_completed_job_triggers_desktop_overlay(tmp_path: Path, monkeypatch) -> cancel_event=threading.Event(), ) - assert overlay.calls == [ - { - "job_id": job_id, - "objective": objective, - "return_message": "Saved todo-demo.txt", - "steps": 11, - "elapsed_seconds": 12.599999999999994, - } - ] - assert db.get_job(job_id)["status"] == "completed" + job = db.get_job(job_id) + assert job is not None + assert job["status"] == "completed" + assert job["return"] == "Saved todo-demo.txt" -def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monkeypatch) -> None: - overlay = _OverlayRecorder() - manager, db, _config = _build_manager(tmp_path, overlay) +def test_non_completed_jobs_are_recorded(tmp_path: Path, monkeypatch) -> None: + manager, db, _config = _build_manager(tmp_path) - failed_job_id = "job_overlay_failed" + failed_job_id = "job_failed" _create_job(db, failed_job_id, "Fail intentionally") failed_result = AgentResult( completed=False, @@ -131,7 +117,6 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke error="Failure", ) monkeypatch.setattr(task_manager_module, "run_job", lambda **_kwargs: (failed_result, _artifacts(tmp_path))) - manager._execute_job( job_id=failed_job_id, objective="Fail intentionally", @@ -155,7 +140,7 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke cancel_event=threading.Event(), ) - cancelled_job_id = "job_overlay_cancelled" + cancelled_job_id = "job_cancelled" _create_job(db, cancelled_job_id, "Cancel intentionally") cancelled_result = AgentResult( completed=False, @@ -170,7 +155,6 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke cancelled=True, ) monkeypatch.setattr(task_manager_module, "run_job", lambda **_kwargs: (cancelled_result, _artifacts(tmp_path))) - manager._execute_job( job_id=cancelled_job_id, objective="Cancel intentionally", @@ -194,13 +178,13 @@ def test_non_completed_jobs_do_not_trigger_desktop_overlay(tmp_path: Path, monke cancel_event=threading.Event(), ) - assert overlay.calls == [] + assert db.get_job(failed_job_id)["status"] == "failed" + assert db.get_job(cancelled_job_id)["status"] == "cancelled" -def test_rejected_job_does_not_trigger_desktop_overlay(tmp_path: Path, monkeypatch) -> None: - overlay = _OverlayRecorder() - manager, db, _config = _build_manager(tmp_path, overlay) - job_id = "job_overlay_rejected" +def test_rejected_job_is_recorded(tmp_path: Path, monkeypatch) -> None: + manager, db, _config = _build_manager(tmp_path) + job_id = "job_rejected" _create_job(db, job_id, "Do something unsafe") monkeypatch.setattr(task_manager_module, "create_openai_client", lambda *_args, **_kwargs: object()) @@ -233,6 +217,33 @@ def test_rejected_job_does_not_trigger_desktop_overlay(tmp_path: Path, monkeypat cancel_event=threading.Event(), ) - assert overlay.calls == [] events = db.get_job_events(job_id) assert events[-1]["event_type"] == "job_rejected" + + +def test_submit_job_rejects_when_another_run_is_active(tmp_path: Path) -> None: + manager, _db, _config = _build_manager(tmp_path) + ready = threading.Event() + release = threading.Event() + + def _hold() -> None: + ready.set() + release.wait() + + active_thread = threading.Thread(target=_hold) + active_thread.start() + ready.wait(1) + manager._running["job_active"] = task_manager_module._RunningJob( + thread=active_thread, + cancel_event=threading.Event(), + started_at="2026-05-30T12:00:00+00:00", + objective="Active", + model="gpt-5.4-mini", + ) + + try: + with pytest.raises(ValueError, match="already active"): + manager.submit_job(objective="Second run") + finally: + release.set() + active_thread.join(timeout=1) diff --git a/tray_service_control.ps1 b/tray_service_control.ps1 deleted file mode 100644 index d77fa7f..0000000 --- a/tray_service_control.ps1 +++ /dev/null @@ -1,53 +0,0 @@ -[CmdletBinding()] -param( - [ValidateSet("start", "stop", "restart")] - [string]$Action, - [string]$ServiceName = "ScreenJobBackend" -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -function Wait-ForStatus { - param( - [Parameter(Mandatory = $true)]$Service, - [Parameter(Mandatory = $true)][System.ServiceProcess.ServiceControllerStatus]$TargetStatus, - [int]$TimeoutSeconds = 20 - ) - - $deadline = (Get-Date).AddSeconds($TimeoutSeconds) - while ((Get-Date) -lt $deadline) { - $Service.Refresh() - if ($Service.Status -eq $TargetStatus) { - return - } - Start-Sleep -Milliseconds 350 - } - - throw "Timed out waiting for service '$($Service.ServiceName)' to reach status '$TargetStatus'." -} - -$service = Get-Service -Name $ServiceName -ErrorAction Stop - -switch ($Action) { - "start" { - if ($service.Status -ne [System.ServiceProcess.ServiceControllerStatus]::Running) { - Start-Service -Name $ServiceName -ErrorAction Stop - Wait-ForStatus -Service $service -TargetStatus ([System.ServiceProcess.ServiceControllerStatus]::Running) - } - } - "stop" { - if ($service.Status -ne [System.ServiceProcess.ServiceControllerStatus]::Stopped) { - Stop-Service -Name $ServiceName -Force -ErrorAction Stop - Wait-ForStatus -Service $service -TargetStatus ([System.ServiceProcess.ServiceControllerStatus]::Stopped) - } - } - "restart" { - if ($service.Status -eq [System.ServiceProcess.ServiceControllerStatus]::Running) { - Restart-Service -Name $ServiceName -Force -ErrorAction Stop - } else { - Start-Service -Name $ServiceName -ErrorAction Stop - } - Wait-ForStatus -Service $service -TargetStatus ([System.ServiceProcess.ServiceControllerStatus]::Running) - } -}